rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pkg/environment: Migrate fs functions to fsx

Browse source
This commit is contained in:
Tom Wiesing 2023-03-02 11:38:30 +01:00
parent 45540ab253
commit 5a43ecfaeb
No known key found for this signature in database
21 changed files with 155 additions and 199 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cmd/dis_ssh.go
View file

@ -1,10 +1,11 @@
package cmd
import (
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/auth"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
gossh "golang.org/x/crypto/ssh"
@ -73,7 +74,7 @@ func (ds disSSH) parseOpts(context wisski_distillery.Context) (user *auth.AuthUs
return nil, nil, err
}
content, err := environment.ReadFile(context.Environment.Environment, ds.Positionals.Path)
content, err := os.ReadFile(ds.Positionals.Path)
if err != nil {
return nil, nil, err
}

6
internal/cli/meta.go
View file

@ -3,12 +3,14 @@ package cli
import (
"errors"
"io/fs"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/bootstrap"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
// metaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable.
@ -41,7 +43,7 @@ func ReadBaseDirectory(env environment.Environment) (value string, err error) {
}
// read the meta config file!
contents, err := environment.ReadFile(env, path)
contents, err := os.ReadFile(path)
if err != nil {
return "", err
}
@ -67,5 +69,5 @@ func WriteBaseDirectory(env environment.Environment, dir string) error {
}
// just put the directory inside it!
return environment.WriteFile(env, path, []byte(dir), fs.ModePerm)
return fsx.WriteFile(path, []byte(dir), fs.ModePerm)
}

5
internal/cli/params.go
View file

@ -19,11 +19,12 @@ type Params struct {
// ParamsFromEnv creates a new set of parameters from the environment.
// Uses [ReadBaseDirectory] or falls back to [BaseDirectoryDefault].
func ParamsFromEnv() (params Params, err error) {
var native environment.Environment
// try to read the base directory!
value, err := ReadBaseDirectory(new(environment.Native)) // TODO: Are we sure about the native environment here?
value, err := ReadBaseDirectory(native) // TODO: Are we sure about the native environment here?
switch {
case environment.IsNotExist(err):
case os.IsNotExist(err):
params.ConfigPath = bootstrap.BaseDirectoryDefault
case err == nil:
params.ConfigPath = value

4
internal/dis/component/backup.go
View file

@ -126,7 +126,7 @@ func (sc *stagingContext) AddDirectory(path string, op func(context.Context) err
}
// run the make directory
if err := sc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil {
if err := fsx.Mkdir(dst, fsx.DefaultDirPerm); err != nil {
return err
}
@ -178,7 +178,7 @@ func (sc *stagingContext) AddFile(path string, op func(ctx context.Context, file
}
// create the file
file, err := sc.env.Create(dst, environment.DefaultFilePerm)
file, err := fsx.Create(dst, fsx.DefaultFilePerm)
if err != nil {
return err
}

3
internal/dis/component/exporter/exporter.go
View file

@ -3,6 +3,7 @@ package exporter
import (
"crypto/rand"
"fmt"
"os"
"path/filepath"
"time"
@ -72,7 +73,7 @@ func (*Exporter) newSnapshotName(prefix string) string {
// NewStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Exporter) NewStagingDir(prefix string) (path string, err error) {
for path == "" || environment.IsExist(err) {
for path == "" || os.IsExist(err) {
path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
err = dis.Still.Environment.Mkdir(path, environment.DefaultFilePerm)
}

2
internal/dis/component/exporter/iface.go
View file

@ -75,7 +75,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
// create the staging directory
logging.LogMessage(progress, ctx, "Creating staging directory")
err = exporter.Environment.Mkdir(stagingDir, environment.DefaultDirPerm)
if !environment.IsExist(err) && err != nil {
if !os.IsExist(err) && err != nil {
return err
}

3
internal/dis/component/exporter/logger/logger.go
View file

@ -8,7 +8,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/internal/status"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/pkglib/collection"
"github.com/tkw1536/pkglib/reflectx"
)
@ -64,7 +63,7 @@ func (log *Logger) Log(ctx context.Context) ([]models.Export, error) {
// partition out the exports that have been deleted!
parts := collection.Partition(exports, func(s models.Export) bool {
_, err := os.Stat(s.Path)
return !environment.IsNotExist(err)
return !os.IsNotExist(err)
})
// go and delete them!

5
internal/dis/component/server/templating/template.go
View file

@ -3,8 +3,7 @@ package templating
import (
_ "embed"
"html/template"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"os"
)
//go:embed "src/footer.html"
@ -16,7 +15,7 @@ func (tpl *Templating) GetCustomizable(dflt *template.Template) *template.Templa
name := dflt.Name()
custom, err := (func() (*template.Template, error) {
data, err := environment.ReadFile(tpl.Environment, tpl.CustomAssetPath(name))
data, err := os.ReadFile(tpl.CustomAssetPath(name))
if err != nil {
return nil, err
}

4
internal/dis/component/ssh2/server_hostkeys.go
View file

@ -65,7 +65,7 @@ func (ssh2 *SSH2) UseOrMakeHostKey(progress io.Writer, ctx context.Context, serv
func (ssh2 *SSH2) ReadOrMakeHostKey(progress io.Writer, ctx context.Context, privateKeyPath string, algorithm HostKeyAlgorithm) (key gossh.Signer, err error) {
hostKey := NewHostKey(algorithm)
if _, e := os.Lstat(privateKeyPath); environment.IsNotExist(e) { // path doesn't exist => generate a new key there!
if _, e := os.Lstat(privateKeyPath); os.IsNotExist(e) { // path doesn't exist => generate a new key there!
err = ssh2.makeHostKey(progress, ctx, hostKey, privateKeyPath)
if err != nil {
err = errors.Wrap(err, "Unable to generate new host key")
@ -84,7 +84,7 @@ func (ssh2 *SSH2) loadHostKey(progress io.Writer, ctx context.Context, key HostK
logging.ProgressF(progress, ctx, "Loading hostkey (algorithm %s) from %q", key.Algorithm(), path)
// read all the bytes from the file
privateKeyBytes, err := environment.ReadFile(ssh2.Environment, path)
privateKeyBytes, err := os.ReadFile(path)
if err != nil {
err = errors.Wrap(err, "Unable to read private key bytes")
return

6
internal/dis/component/stack.go
View file

@ -248,9 +248,9 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co
logging.ProgressF(progress, ctx, "[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = environment.DefaultDirPerm
is.MakeDirsPerm = fsx.DefaultDirPerm
}
if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
if err := fsx.MkdirAll(dst, is.MakeDirsPerm); err != nil {
return err
}
}
@ -279,7 +279,7 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co
dst := filepath.Join(is.Dir, name)
logging.ProgressF(progress, ctx, "[touch] %s\n", dst)
if err := fsx.Touch(env, dst, is.TouchFilesPerm); err != nil {
if err := fsx.Touch(dst, is.TouchFilesPerm); err != nil {
return err
}
}

24
pkg/environment/environment.go
View file

@ -1,25 +1,3 @@
package environment
import (
"io"
"io/fs"
)
// Environment represents an environment that a program can run it.
// It mostly mimics the interfaces of the [os] package.
type Environment interface {
isEnv()
Create(path string, mode fs.FileMode) (WritableFile, error)
Mkdir(path string, mode fs.FileMode) error
MkdirAll(path string, mode fs.FileMode) error
}
type WritableFile interface {
fs.File
io.Writer
}
func init() {
var _ Environment = new(Native)
}
type Environment struct{}

61
pkg/environment/globals.go
View file

@ -1,61 +0,0 @@
package environment
import (
"io"
"io/fs"
"os"
"github.com/tkw1536/pkglib/pools"
)
// DefaultFilePerm is the default mode to use for files
const DefaultFilePerm fs.FileMode = 0666
// DefaultDirPerm is the default mode to use for directories
const DefaultDirPerm fs.FileMode = fs.ModeDir | fs.ModePerm
// IsExist checks if the provided error represents a 'does not exist' errror
func IsExist(err error) bool {
return os.IsExist(err)
}
// IsNotExist checks if the provided error represents a 'does exist' error
func IsNotExist(err error) bool {
return os.IsNotExist(err)
}
// WriteFile is like [os.WriteFile].
func WriteFile(env Environment, path string, data []byte, mode fs.FileMode) error {
handle, err := env.Create(path, mode)
if err != nil {
return err
}
defer handle.Close()
if _, err := handle.Write(data); err != nil {
return err
}
return nil
}
// ReadFile is like [os.ReadFile]
func ReadFile(env Environment, path string) ([]byte, error) {
// open the file!
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// copy everything into a buffer!
buffer := pools.GetBuffer()
defer pools.ReleaseBuffer(buffer)
if _, err := io.Copy(buffer, file); err != nil {
return nil, err
}
// return the buffer contents!
return buffer.Bytes(), nil
}

51
pkg/environment/native_fs.go
View file

@ -1,51 +0,0 @@
package environment
import (
"io/fs"
"os"
"sync"
"syscall"
"time"
)
type Native struct {
ulock sync.Mutex
umask int
}
func (*Native) isEnv() {}
func (n *Native) setMask(umask int) {
n.ulock.Lock()
n.umask = syscall.Umask(umask)
}
func (n *Native) resetMask() {
syscall.Umask(n.umask)
n.ulock.Unlock()
}
func (n *Native) Create(path string, mode fs.FileMode) (WritableFile, error) {
n.ulock.Lock()
defer n.ulock.Unlock()
return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
}
func (*Native) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
func (n *Native) Mkdir(path string, mode fs.FileMode) error {
n.setMask(0)
defer n.resetMask()
return os.Mkdir(path, fs.ModeDir|mode)
}
func (n *Native) MkdirAll(path string, mode fs.FileMode) error {
n.setMask(0)
defer n.resetMask()
return os.MkdirAll(path, fs.ModeDir|mode)
}

8
pkg/fsx/copy.go
View file

@ -18,6 +18,8 @@ var ErrCopySameFile = errors.New("src and dst must be different")
//
// When dst and src are the same file, returns ErrCopySameFile.
// When ctx is closed, the file is not copied.
//
// Ignores the umask.
func CopyFile(ctx context.Context, env environment.Environment, dst, src string) error {
if err := ctx.Err(); err != nil {
return err
@ -41,7 +43,7 @@ func CopyFile(ctx context.Context, env environment.Environment, dst, src string)
}
// open or create the destination
dstFile, err := env.Create(dst, srcStat.Mode())
dstFile, err := Create(dst, srcStat.Mode())
if err != nil {
return err
}
@ -141,8 +143,8 @@ func CopyDirectory(ctx context.Context, env environment.Environment, dst, src st
// create the directory, but ignore an error if the directory already exists.
// this is so that we can copy one tree into another tree.
err = env.Mkdir(dst, info.Mode())
if environment.IsExist(err) && IsDirectory(env, dst) {
err = Mkdir(dst, info.Mode())
if os.IsExist(err) && IsDirectory(env, dst) {
err = nil
}

16
pkg/fsx/fsx.go Normal file
View file

@ -0,0 +1,16 @@
// Package fsx provides additional file system functionality.
//
// Several functions in this package provide umask-ignoring functions.
// Using these functions intervenes with the global umask.
//
// It is not safe to use functions provided by the standard go library concurrently with this function.
// Users should take care that no other code in their application uses these functions.
package fsx
import "io/fs"
// DefaultFilePerm should be used by callers to use a consistent file mode for new files.
const DefaultFilePerm fs.FileMode = 0666
// DefaultDirPerm should be used by callers to use a consistent mode for new directories.
const DefaultDirPerm fs.FileMode = fs.ModeDir | fs.ModePerm

2
pkg/fsx/same.go
View file

@ -62,7 +62,7 @@ func couldBeSameFile(env environment.Environment, path1, path2 string) (same, au
}
// only 1 file does not exist => they could be different
if environment.IsNotExist(err1) != environment.IsNotExist(err2) {
if os.IsNotExist(err1) != os.IsNotExist(err2) {
return
}
}

35
pkg/fsx/touch.go
View file

@ -1,35 +0,0 @@
package fsx
import (
"io/fs"
"os"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Touch touches a file.
// It is similar to the unix 'touch' command.
//
// If the file does not exist exists, it is created using [env.Create].
// If the file does exist, it's access and modification times are updated to the current time.
func Touch(env environment.Environment, path string, perm fs.FileMode) error {
if perm == 0 {
perm = environment.DefaultFilePerm
}
_, err := os.Stat(path)
switch {
case environment.IsNotExist(err):
f, err := env.Create(path, perm)
if err != nil {
return err
}
defer f.Close()
return nil
case err != nil:
return err
default:
now := time.Now().Local()
return os.Chtimes(path, now, now)
}
}

101
pkg/fsx/unmasked.go Normal file
View file

@ -0,0 +1,101 @@
package fsx
import (
"io/fs"
"os"
"sync"
"syscall"
"time"
)
// mask is the global mask lock
var m mask
// mask allows disabling and re-enabling the global umask.
// it is used by allow functions of this package.
type mask struct {
l sync.Mutex // locked?
umask int // previous mask
}
// Lock blocks until no other function is using this umask
// and then sets it to 0.
func (mask *mask) Lock() {
mask.l.Lock()
mask.umask = syscall.Umask(0)
}
func (mask *mask) Unlock() {
mask.umask = syscall.Umask(mask.umask)
mask.l.Unlock()
}
// WriteFile is like [os.WriteFile], but ignores the umask.
func WriteFile(path string, data []byte, mode fs.FileMode) error {
handle, err := Create(path, mode)
if err != nil {
return err
}
defer handle.Close()
if _, err := handle.Write(data); err != nil {
return err
}
return nil
}
// Create creates a new file with the given mode.
// This function ignores the umask.
func Create(path string, mode fs.FileMode) (*os.File, error) {
m.Lock()
defer m.Unlock()
return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
}
// Mkdir creates a new directory with the given mode.
// This function ignores the umask.
func Mkdir(path string, mode fs.FileMode) error {
m.Lock()
defer m.Unlock()
return os.Mkdir(path, fs.ModeDir|mode)
}
// MkdirAll creates a new directory and all potentially missing parent directories.
// This function ignores the umask.
func MkdirAll(path string, mode fs.FileMode) error {
m.Lock()
defer m.Unlock()
return os.MkdirAll(path, fs.ModeDir|mode)
}
// Touch touches a file.
// It is similar to the unix 'touch' command.
//
// If the file does not exist exists, it is created using [Create].
// If the file does exist, it's access and modification times are updated to the current time.
//
// This function ignores the umask.
func Touch(path string, perm fs.FileMode) error {
if perm == 0 {
perm = DefaultFilePerm
}
_, err := os.Stat(path)
switch {
case os.IsNotExist(err):
f, err := Create(path, perm)
if err != nil {
return err
}
defer f.Close()
return nil
case err != nil:
return err
default:
now := time.Now().Local()
return os.Chtimes(path, now, now)
}
}

3
pkg/targz/targz.go
View file

@ -10,6 +10,7 @@ import (
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
// Package packages the source directory into a 'tar.gz' file into destination.
@ -18,7 +19,7 @@ import (
// onCopy, when not nil, is called for each file being copied into the archive.
func Package(env environment.Environment, dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
// create the target archive
archive, err := env.Create(dst, environment.DefaultFilePerm)
archive, err := fsx.Create(dst, fsx.DefaultFilePerm)
if err != nil {
return 0, err
}

7
pkg/unpack/resource.go
View file

@ -7,6 +7,7 @@ import (
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
)
@ -83,8 +84,8 @@ func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, sr
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case environment.IsNotExist(dstErr):
if err := env.MkdirAll(dst, srcInfo.Mode()); err != nil {
case os.IsNotExist(dstErr):
if err := fsx.MkdirAll(dst, srcInfo.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
@ -121,7 +122,7 @@ func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, sr
func installFile(env environment.Environment, dst string, srcInfo fs.FileInfo, src fs.File) error {
// create the file using the right mode!
file, err := env.Create(dst, srcInfo.Mode())
file, err := fsx.Create(dst, srcInfo.Mode())
if err != nil {
return err
}

3
pkg/unpack/template.go
View file

@ -9,6 +9,7 @@ import (
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
@ -222,7 +223,7 @@ func InstallTemplate(env environment.Environment, dst string, context map[string
}
// open the destination file
file, err := env.Create(dst, srcInfo.Mode())
file, err := fsx.Create(dst, srcInfo.Mode())
if err != nil {
return err
}