From 5a43ecfaebbdeb2fbba4e32ea909031d2989bff0 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Thu, 2 Mar 2023 11:38:30 +0100 Subject: [PATCH] pkg/environment: Migrate fs functions to fsx --- cmd/dis_ssh.go | 5 +- internal/cli/meta.go | 6 +- internal/cli/params.go | 5 +- internal/dis/component/backup.go | 4 +- internal/dis/component/exporter/exporter.go | 3 +- internal/dis/component/exporter/iface.go | 2 +- .../dis/component/exporter/logger/logger.go | 3 +- .../component/server/templating/template.go | 5 +- .../dis/component/ssh2/server_hostkeys.go | 4 +- internal/dis/component/stack.go | 6 +- pkg/environment/environment.go | 24 +---- pkg/environment/globals.go | 61 ----------- pkg/environment/native_fs.go | 51 --------- pkg/fsx/copy.go | 8 +- pkg/fsx/fsx.go | 16 +++ pkg/fsx/same.go | 2 +- pkg/fsx/touch.go | 35 ------ pkg/fsx/unmasked.go | 101 ++++++++++++++++++ pkg/targz/targz.go | 3 +- pkg/unpack/resource.go | 7 +- pkg/unpack/template.go | 3 +- 21 files changed, 155 insertions(+), 199 deletions(-) delete mode 100644 pkg/environment/globals.go delete mode 100644 pkg/environment/native_fs.go create mode 100644 pkg/fsx/fsx.go delete mode 100644 pkg/fsx/touch.go create mode 100644 pkg/fsx/unmasked.go diff --git a/cmd/dis_ssh.go b/cmd/dis_ssh.go index dcd5f23..b0ff60e 100644 --- a/cmd/dis_ssh.go +++ b/cmd/dis_ssh.go @@ -1,10 +1,11 @@ package cmd import ( + "os" + wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/cli" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/auth" - "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/goprogram/exit" gossh "golang.org/x/crypto/ssh" @@ -73,7 +74,7 @@ func (ds disSSH) parseOpts(context wisski_distillery.Context) (user *auth.AuthUs return nil, nil, err } - content, err := environment.ReadFile(context.Environment.Environment, ds.Positionals.Path) + content, err := os.ReadFile(ds.Positionals.Path) if err != nil { return nil, nil, err } diff --git a/internal/cli/meta.go b/internal/cli/meta.go index 057abd6..915172c 100644 --- a/internal/cli/meta.go +++ b/internal/cli/meta.go @@ -3,12 +3,14 @@ package cli import ( "errors" "io/fs" + "os" "os/user" "path/filepath" "strings" "github.com/FAU-CDI/wisski-distillery/internal/bootstrap" "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" ) // metaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable. @@ -41,7 +43,7 @@ func ReadBaseDirectory(env environment.Environment) (value string, err error) { } // read the meta config file! - contents, err := environment.ReadFile(env, path) + contents, err := os.ReadFile(path) if err != nil { return "", err } @@ -67,5 +69,5 @@ func WriteBaseDirectory(env environment.Environment, dir string) error { } // just put the directory inside it! - return environment.WriteFile(env, path, []byte(dir), fs.ModePerm) + return fsx.WriteFile(path, []byte(dir), fs.ModePerm) } diff --git a/internal/cli/params.go b/internal/cli/params.go index 99b721d..cbf0abc 100644 --- a/internal/cli/params.go +++ b/internal/cli/params.go @@ -19,11 +19,12 @@ type Params struct { // ParamsFromEnv creates a new set of parameters from the environment. // Uses [ReadBaseDirectory] or falls back to [BaseDirectoryDefault]. func ParamsFromEnv() (params Params, err error) { + var native environment.Environment // try to read the base directory! - value, err := ReadBaseDirectory(new(environment.Native)) // TODO: Are we sure about the native environment here? + value, err := ReadBaseDirectory(native) // TODO: Are we sure about the native environment here? switch { - case environment.IsNotExist(err): + case os.IsNotExist(err): params.ConfigPath = bootstrap.BaseDirectoryDefault case err == nil: params.ConfigPath = value diff --git a/internal/dis/component/backup.go b/internal/dis/component/backup.go index 54375aa..ffc954a 100644 --- a/internal/dis/component/backup.go +++ b/internal/dis/component/backup.go @@ -126,7 +126,7 @@ func (sc *stagingContext) AddDirectory(path string, op func(context.Context) err } // run the make directory - if err := sc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil { + if err := fsx.Mkdir(dst, fsx.DefaultDirPerm); err != nil { return err } @@ -178,7 +178,7 @@ func (sc *stagingContext) AddFile(path string, op func(ctx context.Context, file } // create the file - file, err := sc.env.Create(dst, environment.DefaultFilePerm) + file, err := fsx.Create(dst, fsx.DefaultFilePerm) if err != nil { return err } diff --git a/internal/dis/component/exporter/exporter.go b/internal/dis/component/exporter/exporter.go index aaf34ab..4ad08d1 100644 --- a/internal/dis/component/exporter/exporter.go +++ b/internal/dis/component/exporter/exporter.go @@ -3,6 +3,7 @@ package exporter import ( "crypto/rand" "fmt" + "os" "path/filepath" "time" @@ -72,7 +73,7 @@ func (*Exporter) newSnapshotName(prefix string) string { // NewStagingDir returns the path to a new snapshot directory. // The directory is guaranteed to have been freshly created. func (dis *Exporter) NewStagingDir(prefix string) (path string, err error) { - for path == "" || environment.IsExist(err) { + for path == "" || os.IsExist(err) { path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix)) err = dis.Still.Environment.Mkdir(path, environment.DefaultFilePerm) } diff --git a/internal/dis/component/exporter/iface.go b/internal/dis/component/exporter/iface.go index 9b21a0a..38fbfac 100644 --- a/internal/dis/component/exporter/iface.go +++ b/internal/dis/component/exporter/iface.go @@ -75,7 +75,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta // create the staging directory logging.LogMessage(progress, ctx, "Creating staging directory") err = exporter.Environment.Mkdir(stagingDir, environment.DefaultDirPerm) - if !environment.IsExist(err) && err != nil { + if !os.IsExist(err) && err != nil { return err } diff --git a/internal/dis/component/exporter/logger/logger.go b/internal/dis/component/exporter/logger/logger.go index 3ca855e..01c71f5 100644 --- a/internal/dis/component/exporter/logger/logger.go +++ b/internal/dis/component/exporter/logger/logger.go @@ -8,7 +8,6 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql" "github.com/FAU-CDI/wisski-distillery/internal/models" "github.com/FAU-CDI/wisski-distillery/internal/status" - "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/pkglib/collection" "github.com/tkw1536/pkglib/reflectx" ) @@ -64,7 +63,7 @@ func (log *Logger) Log(ctx context.Context) ([]models.Export, error) { // partition out the exports that have been deleted! parts := collection.Partition(exports, func(s models.Export) bool { _, err := os.Stat(s.Path) - return !environment.IsNotExist(err) + return !os.IsNotExist(err) }) // go and delete them! diff --git a/internal/dis/component/server/templating/template.go b/internal/dis/component/server/templating/template.go index 0eb18b1..d2c4807 100644 --- a/internal/dis/component/server/templating/template.go +++ b/internal/dis/component/server/templating/template.go @@ -3,8 +3,7 @@ package templating import ( _ "embed" "html/template" - - "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "os" ) //go:embed "src/footer.html" @@ -16,7 +15,7 @@ func (tpl *Templating) GetCustomizable(dflt *template.Template) *template.Templa name := dflt.Name() custom, err := (func() (*template.Template, error) { - data, err := environment.ReadFile(tpl.Environment, tpl.CustomAssetPath(name)) + data, err := os.ReadFile(tpl.CustomAssetPath(name)) if err != nil { return nil, err } diff --git a/internal/dis/component/ssh2/server_hostkeys.go b/internal/dis/component/ssh2/server_hostkeys.go index 78dc3ee..0c20b44 100644 --- a/internal/dis/component/ssh2/server_hostkeys.go +++ b/internal/dis/component/ssh2/server_hostkeys.go @@ -65,7 +65,7 @@ func (ssh2 *SSH2) UseOrMakeHostKey(progress io.Writer, ctx context.Context, serv func (ssh2 *SSH2) ReadOrMakeHostKey(progress io.Writer, ctx context.Context, privateKeyPath string, algorithm HostKeyAlgorithm) (key gossh.Signer, err error) { hostKey := NewHostKey(algorithm) - if _, e := os.Lstat(privateKeyPath); environment.IsNotExist(e) { // path doesn't exist => generate a new key there! + if _, e := os.Lstat(privateKeyPath); os.IsNotExist(e) { // path doesn't exist => generate a new key there! err = ssh2.makeHostKey(progress, ctx, hostKey, privateKeyPath) if err != nil { err = errors.Wrap(err, "Unable to generate new host key") @@ -84,7 +84,7 @@ func (ssh2 *SSH2) loadHostKey(progress io.Writer, ctx context.Context, key HostK logging.ProgressF(progress, ctx, "Loading hostkey (algorithm %s) from %q", key.Algorithm(), path) // read all the bytes from the file - privateKeyBytes, err := environment.ReadFile(ssh2.Environment, path) + privateKeyBytes, err := os.ReadFile(path) if err != nil { err = errors.Wrap(err, "Unable to read private key bytes") return diff --git a/internal/dis/component/stack.go b/internal/dis/component/stack.go index 8805a92..95e4b22 100644 --- a/internal/dis/component/stack.go +++ b/internal/dis/component/stack.go @@ -248,9 +248,9 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co logging.ProgressF(progress, ctx, "[make] %s\n", dst) if is.MakeDirsPerm == fs.FileMode(0) { - is.MakeDirsPerm = environment.DefaultDirPerm + is.MakeDirsPerm = fsx.DefaultDirPerm } - if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil { + if err := fsx.MkdirAll(dst, is.MakeDirsPerm); err != nil { return err } } @@ -279,7 +279,7 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co dst := filepath.Join(is.Dir, name) logging.ProgressF(progress, ctx, "[touch] %s\n", dst) - if err := fsx.Touch(env, dst, is.TouchFilesPerm); err != nil { + if err := fsx.Touch(dst, is.TouchFilesPerm); err != nil { return err } } diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 439f608..4ba3f00 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -1,25 +1,3 @@ package environment -import ( - "io" - "io/fs" -) - -// Environment represents an environment that a program can run it. -// It mostly mimics the interfaces of the [os] package. -type Environment interface { - isEnv() - - Create(path string, mode fs.FileMode) (WritableFile, error) - Mkdir(path string, mode fs.FileMode) error - MkdirAll(path string, mode fs.FileMode) error -} - -type WritableFile interface { - fs.File - io.Writer -} - -func init() { - var _ Environment = new(Native) -} +type Environment struct{} diff --git a/pkg/environment/globals.go b/pkg/environment/globals.go deleted file mode 100644 index 511552e..0000000 --- a/pkg/environment/globals.go +++ /dev/null @@ -1,61 +0,0 @@ -package environment - -import ( - "io" - "io/fs" - "os" - - "github.com/tkw1536/pkglib/pools" -) - -// DefaultFilePerm is the default mode to use for files -const DefaultFilePerm fs.FileMode = 0666 - -// DefaultDirPerm is the default mode to use for directories -const DefaultDirPerm fs.FileMode = fs.ModeDir | fs.ModePerm - -// IsExist checks if the provided error represents a 'does not exist' errror -func IsExist(err error) bool { - return os.IsExist(err) -} - -// IsNotExist checks if the provided error represents a 'does exist' error -func IsNotExist(err error) bool { - return os.IsNotExist(err) -} - -// WriteFile is like [os.WriteFile]. -func WriteFile(env Environment, path string, data []byte, mode fs.FileMode) error { - handle, err := env.Create(path, mode) - if err != nil { - return err - } - defer handle.Close() - - if _, err := handle.Write(data); err != nil { - return err - } - - return nil -} - -// ReadFile is like [os.ReadFile] -func ReadFile(env Environment, path string) ([]byte, error) { - // open the file! - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - // copy everything into a buffer! - buffer := pools.GetBuffer() - defer pools.ReleaseBuffer(buffer) - - if _, err := io.Copy(buffer, file); err != nil { - return nil, err - } - - // return the buffer contents! - return buffer.Bytes(), nil -} diff --git a/pkg/environment/native_fs.go b/pkg/environment/native_fs.go deleted file mode 100644 index bedc5ac..0000000 --- a/pkg/environment/native_fs.go +++ /dev/null @@ -1,51 +0,0 @@ -package environment - -import ( - "io/fs" - "os" - "sync" - "syscall" - "time" -) - -type Native struct { - ulock sync.Mutex - umask int -} - -func (*Native) isEnv() {} - -func (n *Native) setMask(umask int) { - n.ulock.Lock() - n.umask = syscall.Umask(umask) -} - -func (n *Native) resetMask() { - syscall.Umask(n.umask) - n.ulock.Unlock() -} - -func (n *Native) Create(path string, mode fs.FileMode) (WritableFile, error) { - n.ulock.Lock() - defer n.ulock.Unlock() - - return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) -} - -func (*Native) Chtimes(name string, atime time.Time, mtime time.Time) error { - return os.Chtimes(name, atime, mtime) -} - -func (n *Native) Mkdir(path string, mode fs.FileMode) error { - n.setMask(0) - defer n.resetMask() - - return os.Mkdir(path, fs.ModeDir|mode) -} - -func (n *Native) MkdirAll(path string, mode fs.FileMode) error { - n.setMask(0) - defer n.resetMask() - - return os.MkdirAll(path, fs.ModeDir|mode) -} diff --git a/pkg/fsx/copy.go b/pkg/fsx/copy.go index d049ac3..9363a98 100644 --- a/pkg/fsx/copy.go +++ b/pkg/fsx/copy.go @@ -18,6 +18,8 @@ var ErrCopySameFile = errors.New("src and dst must be different") // // When dst and src are the same file, returns ErrCopySameFile. // When ctx is closed, the file is not copied. +// +// Ignores the umask. func CopyFile(ctx context.Context, env environment.Environment, dst, src string) error { if err := ctx.Err(); err != nil { return err @@ -41,7 +43,7 @@ func CopyFile(ctx context.Context, env environment.Environment, dst, src string) } // open or create the destination - dstFile, err := env.Create(dst, srcStat.Mode()) + dstFile, err := Create(dst, srcStat.Mode()) if err != nil { return err } @@ -141,8 +143,8 @@ func CopyDirectory(ctx context.Context, env environment.Environment, dst, src st // create the directory, but ignore an error if the directory already exists. // this is so that we can copy one tree into another tree. - err = env.Mkdir(dst, info.Mode()) - if environment.IsExist(err) && IsDirectory(env, dst) { + err = Mkdir(dst, info.Mode()) + if os.IsExist(err) && IsDirectory(env, dst) { err = nil } diff --git a/pkg/fsx/fsx.go b/pkg/fsx/fsx.go new file mode 100644 index 0000000..2c15c2d --- /dev/null +++ b/pkg/fsx/fsx.go @@ -0,0 +1,16 @@ +// Package fsx provides additional file system functionality. +// +// Several functions in this package provide umask-ignoring functions. +// Using these functions intervenes with the global umask. +// +// It is not safe to use functions provided by the standard go library concurrently with this function. +// Users should take care that no other code in their application uses these functions. +package fsx + +import "io/fs" + +// DefaultFilePerm should be used by callers to use a consistent file mode for new files. +const DefaultFilePerm fs.FileMode = 0666 + +// DefaultDirPerm should be used by callers to use a consistent mode for new directories. +const DefaultDirPerm fs.FileMode = fs.ModeDir | fs.ModePerm diff --git a/pkg/fsx/same.go b/pkg/fsx/same.go index 719ebc2..7721234 100644 --- a/pkg/fsx/same.go +++ b/pkg/fsx/same.go @@ -62,7 +62,7 @@ func couldBeSameFile(env environment.Environment, path1, path2 string) (same, au } // only 1 file does not exist => they could be different - if environment.IsNotExist(err1) != environment.IsNotExist(err2) { + if os.IsNotExist(err1) != os.IsNotExist(err2) { return } } diff --git a/pkg/fsx/touch.go b/pkg/fsx/touch.go deleted file mode 100644 index d261d52..0000000 --- a/pkg/fsx/touch.go +++ /dev/null @@ -1,35 +0,0 @@ -package fsx - -import ( - "io/fs" - "os" - "time" - - "github.com/FAU-CDI/wisski-distillery/pkg/environment" -) - -// Touch touches a file. -// It is similar to the unix 'touch' command. -// -// If the file does not exist exists, it is created using [env.Create]. -// If the file does exist, it's access and modification times are updated to the current time. -func Touch(env environment.Environment, path string, perm fs.FileMode) error { - if perm == 0 { - perm = environment.DefaultFilePerm - } - _, err := os.Stat(path) - switch { - case environment.IsNotExist(err): - f, err := env.Create(path, perm) - if err != nil { - return err - } - defer f.Close() - return nil - case err != nil: - return err - default: - now := time.Now().Local() - return os.Chtimes(path, now, now) - } -} diff --git a/pkg/fsx/unmasked.go b/pkg/fsx/unmasked.go new file mode 100644 index 0000000..8602012 --- /dev/null +++ b/pkg/fsx/unmasked.go @@ -0,0 +1,101 @@ +package fsx + +import ( + "io/fs" + "os" + "sync" + "syscall" + "time" +) + +// mask is the global mask lock +var m mask + +// mask allows disabling and re-enabling the global umask. +// it is used by allow functions of this package. +type mask struct { + l sync.Mutex // locked? + umask int // previous mask +} + +// Lock blocks until no other function is using this umask +// and then sets it to 0. +func (mask *mask) Lock() { + mask.l.Lock() + mask.umask = syscall.Umask(0) +} + +func (mask *mask) Unlock() { + mask.umask = syscall.Umask(mask.umask) + mask.l.Unlock() +} + +// WriteFile is like [os.WriteFile], but ignores the umask. +func WriteFile(path string, data []byte, mode fs.FileMode) error { + handle, err := Create(path, mode) + if err != nil { + return err + } + defer handle.Close() + + if _, err := handle.Write(data); err != nil { + return err + } + + return nil +} + +// Create creates a new file with the given mode. +// This function ignores the umask. +func Create(path string, mode fs.FileMode) (*os.File, error) { + m.Lock() + defer m.Unlock() + + return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) +} + +// Mkdir creates a new directory with the given mode. +// This function ignores the umask. +func Mkdir(path string, mode fs.FileMode) error { + m.Lock() + defer m.Unlock() + + return os.Mkdir(path, fs.ModeDir|mode) +} + +// MkdirAll creates a new directory and all potentially missing parent directories. +// This function ignores the umask. +func MkdirAll(path string, mode fs.FileMode) error { + m.Lock() + defer m.Unlock() + + return os.MkdirAll(path, fs.ModeDir|mode) +} + +// Touch touches a file. +// It is similar to the unix 'touch' command. +// +// If the file does not exist exists, it is created using [Create]. +// If the file does exist, it's access and modification times are updated to the current time. +// +// This function ignores the umask. +func Touch(path string, perm fs.FileMode) error { + if perm == 0 { + perm = DefaultFilePerm + } + _, err := os.Stat(path) + switch { + case os.IsNotExist(err): + f, err := Create(path, perm) + if err != nil { + return err + } + defer f.Close() + return nil + case err != nil: + return err + default: + now := time.Now().Local() + return os.Chtimes(path, now, now) + } +} diff --git a/pkg/targz/targz.go b/pkg/targz/targz.go index ac5aa80..ea3e188 100644 --- a/pkg/targz/targz.go +++ b/pkg/targz/targz.go @@ -10,6 +10,7 @@ import ( "path/filepath" "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" ) // Package packages the source directory into a 'tar.gz' file into destination. @@ -18,7 +19,7 @@ import ( // onCopy, when not nil, is called for each file being copied into the archive. func Package(env environment.Environment, dst, src string, onCopy func(rel string, src string)) (count int64, err error) { // create the target archive - archive, err := env.Create(dst, environment.DefaultFilePerm) + archive, err := fsx.Create(dst, fsx.DefaultFilePerm) if err != nil { return 0, err } diff --git a/pkg/unpack/resource.go b/pkg/unpack/resource.go index 20f504f..319aaf5 100644 --- a/pkg/unpack/resource.go +++ b/pkg/unpack/resource.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/pkg/errors" ) @@ -83,8 +84,8 @@ func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, sr // create the destination dstStat, dstErr := os.Stat(dst) switch { - case environment.IsNotExist(dstErr): - if err := env.MkdirAll(dst, srcInfo.Mode()); err != nil { + case os.IsNotExist(dstErr): + if err := fsx.MkdirAll(dst, srcInfo.Mode()); err != nil { return errors.Wrapf(err, "Error creating destination directory %s", dst) } case dstErr != nil: @@ -121,7 +122,7 @@ func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, sr func installFile(env environment.Environment, dst string, srcInfo fs.FileInfo, src fs.File) error { // create the file using the right mode! - file, err := env.Create(dst, srcInfo.Mode()) + file, err := fsx.Create(dst, srcInfo.Mode()) if err != nil { return err } diff --git a/pkg/unpack/template.go b/pkg/unpack/template.go index 98e17d2..a04271c 100644 --- a/pkg/unpack/template.go +++ b/pkg/unpack/template.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -222,7 +223,7 @@ func InstallTemplate(env environment.Environment, dst string, context map[string } // open the destination file - file, err := env.Create(dst, srcInfo.Mode()) + file, err := fsx.Create(dst, srcInfo.Mode()) if err != nil { return err }