diff --git a/TODO.md b/TODO.md index dc8e3d9..aa4567c 100644 --- a/TODO.md +++ b/TODO.md @@ -5,12 +5,10 @@ - Why a factory? - First steps after provisioning - +- Use `environment.Dial()` and `environment.Listen()` - Move `provision_entrypoint.sh` into go - Enhance Snapshots - Export the Docker Images - Avoid running `docker compose` executable and shift it to a library - Move resolver code into this -- Cleanup code: Have consistent error handling -- Add a proper metadata / statistics server - Single Malt Mode: Support having a single instance only! \ No newline at end of file diff --git a/cmd/backup.go b/cmd/backup.go index 80c30f9..0c9f668 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -1,12 +1,10 @@ package cmd import ( - "io/fs" - "os" - wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/backup" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/targz" "github.com/tkw1536/goprogram/exit" @@ -59,7 +57,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error { } defer func() { logging.LogMessage(context.IOStream, "Removing snapshot staging directory") - os.RemoveAll(sPath) + dis.Environment.RemoveAll(sPath) }() } else { // staging mode: use dest as a destination @@ -73,8 +71,8 @@ func (bk backupC) Run(context wisski_distillery.Context) error { // create the directory (if it doesn't already exist) logging.LogMessage(context.IOStream, "Creating staging directory") - err = os.Mkdir(sPath, fs.ModePerm) - if !os.IsExist(err) && err != nil { + err = dis.Core.Environment.Mkdir(sPath, environment.DefaultDirPerm) + if !environment.IsExist(err) && err != nil { return errSnapshotFailed.WithMessageF(err) } err = nil @@ -86,7 +84,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error { Dest: sPath, Auto: bk.Positionals.Dest == "", }) - backup.WriteReport(context.IOStream) + backup.WriteReport(dis.Core.Environment, context.IOStream) return nil }, context.IOStream, "Generating Backup") @@ -108,7 +106,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error { if err := logging.LogOperation(func() error { context.IOStream.Println(archivePath) - count, err = targz.Package(archivePath, sPath, func(dst, src string) { + count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) { context.Printf("\033[2K\r%s", dst) }) context.Println("") diff --git a/cmd/blind_update.go b/cmd/blind_update.go index b92f9bf..5d9dd82 100644 --- a/cmd/blind_update.go +++ b/cmd/blind_update.go @@ -3,7 +3,7 @@ package cmd import ( wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/core" - "github.com/FAU-CDI/wisski-distillery/pkg/execx" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/goprogram/exit" ) @@ -47,7 +47,7 @@ func (bu blindUpdate) Run(context wisski_distillery.Context) error { code, err := instance.Shell(context.IOStream, "/runtime/blind_update.sh") if err != nil { - return errBlindUpdateFailed.WithMessageF(instance.Slug, execx.ExecCommandError) + return errBlindUpdateFailed.WithMessageF(instance.Slug, environment.ExecCommandError) } if code != 0 { return errBlindUpdateFailed.WithMessageF(instance.Slug, code) diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 330bc45..60b869a 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -2,12 +2,12 @@ package cmd import ( "io/fs" - "os" "path/filepath" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/config" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/tkw1536/goprogram/exit" @@ -67,11 +67,15 @@ var errBootstrapCreateFile = exit.Error{ } func (bs bootstrap) Run(context wisski_distillery.Context) error { + // installation environment is the native environment! + // TODO: Should this be configurable? + var env environment.Native + root := bs.Directory // check that we didn't get a different base directory { - got, err := core.ReadBaseDirectory() + got, err := core.ReadBaseDirectory(env) if err == nil && got != "" && got != root { return errBootstrapDifferent.WithMessageF(got) } @@ -79,10 +83,10 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { { logging.LogMessage(context.IOStream, "Creating root deployment directory") - if err := os.MkdirAll(root, fs.ModeDir); err != nil { + if err := env.MkdirAll(root, environment.DefaultDirPerm); err != nil { return errBootstrapFailedToCreateDirectory.WithMessageF(root) } - if err := core.WriteBaseDirectory(root); err != nil { + if err := core.WriteBaseDirectory(env, root); err != nil { return errBootstrapFailedToSaveDirectory.WithMessageF(root) } context.Println(root) @@ -98,18 +102,18 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { tpl.DefaultDomain = bs.Hostname // and use thge defaults - if err := tpl.SetDefaults(); err != nil { + if err := tpl.SetDefaults(env); err != nil { return errBootstrapWriteConfig.WithMessageF(err) } { logging.LogMessage(context.IOStream, "Copying over wdcli executable") - exe, err := os.Executable() + exe, err := env.Executable() if err != nil { return errBoostrapFailedToCopyExe.WithMessageF(err) } - err = fsx.CopyFile(wdcliPath, exe) + err = fsx.CopyFile(env, wdcliPath, exe) if err != nil && err != fsx.ErrCopySameFile { return errBoostrapFailedToCopyExe.WithMessageF(err) } @@ -117,9 +121,9 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { } { - if !fsx.IsFile(envPath) { + if !fsx.IsFile(env, envPath) { if err := logging.LogOperation(func() error { - env, err := os.Create(envPath) + env, err := env.Create(envPath, environment.DefaultFilePerm) if err != nil { return err } @@ -133,7 +137,8 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { if err := logging.LogOperation(func() error { context.Println(tpl.SelfOverridesFile) - if err := os.WriteFile( + if err := environment.WriteFile( + env, tpl.SelfOverridesFile, core.DefaultOverridesJSON, fs.ModePerm, @@ -142,7 +147,8 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { } context.Println(tpl.AuthorizedKeys) - if err := os.WriteFile( + if err := environment.WriteFile( + env, tpl.AuthorizedKeys, core.DefaultAuthorizedKeys, fs.ModePerm, @@ -160,14 +166,14 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { // re-read the configuration and print it! logging.LogMessage(context.IOStream, "Configuration is now complete") - f, err := os.Open(envPath) + f, err := env.Open(envPath) if err != nil { return errBootstrapOpenConfig.WithMessageF(err) } defer f.Close() var cfg config.Config - if err := cfg.Unmarshal(f); err != nil { + if err := cfg.Unmarshal(env, f); err != nil { return errBootstrapOpenConfig.WithMessageF(err) } context.Println(cfg) diff --git a/cmd/monday.go b/cmd/monday.go index e62e525..a7b0f67 100644 --- a/cmd/monday.go +++ b/cmd/monday.go @@ -1,10 +1,10 @@ package cmd import ( - "os" - wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" ) @@ -29,13 +29,10 @@ func (monday) Description() wisski_distillery.Description { } func (monday monday) AfterParse() error { - _, err := os.Stat(monday.Positionals.GraphdbZip) - if os.IsNotExist(err) { + // TODO: Use a generic environment here! + if !fsx.IsFile(environment.Native{}, monday.Positionals.GraphdbZip) { return errNoGraphDBZip.WithMessageF(monday.Positionals.GraphdbZip) } - if err != nil { - return err - } return nil } diff --git a/cmd/provision.go b/cmd/provision.go index 4243b7b..ff54a71 100644 --- a/cmd/provision.go +++ b/cmd/provision.go @@ -57,7 +57,7 @@ func (p provision) Run(context wisski_distillery.Context) error { // check that the base directory does not exist logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase) - if fsx.IsDirectory(instance.FilesystemBase) { + if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) { return errProvisionAlreadyExists.WithMessageF(slug) } diff --git a/cmd/purge.go b/cmd/purge.go index bb65818..39f6faf 100644 --- a/cmd/purge.go +++ b/cmd/purge.go @@ -1,8 +1,6 @@ package cmd import ( - "os" - wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/component/instances" "github.com/FAU-CDI/wisski-distillery/internal/core" @@ -73,7 +71,7 @@ func (p purge) Run(context wisski_distillery.Context) error { // remove the filesystem logging.LogMessage(context.IOStream, "Removing from filesystem %s", instance.FilesystemBase) - if err := os.RemoveAll(instance.FilesystemBase); err != nil { + if err := dis.Core.Environment.RemoveAll(instance.FilesystemBase); err != nil { context.EPrintln(err) } diff --git a/cmd/rebuild.go b/cmd/rebuild.go index 88cb704..8dbbfba 100644 --- a/cmd/rebuild.go +++ b/cmd/rebuild.go @@ -33,7 +33,9 @@ var errRebuildFailed = exit.Error{ } func (rb rebuild) Run(context wisski_distillery.Context) error { - instances, err := context.Environment.Instances().Load(rb.Positionals.Slug...) + dis := context.Environment + + instances, err := dis.Instances().Load(rb.Positionals.Slug...) if err != nil { return err } @@ -44,7 +46,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error { logging.LogOperation(func() error { s := instance.Barrel() if err := logging.LogOperation(func() error { - return s.Install(context.IOStream, component.InstallationContext{}) + return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{}) }, context.IOStream, "Installing docker stack"); err != nil { globalErr = err return err diff --git a/cmd/reserve.go b/cmd/reserve.go index 997748d..40fd05d 100644 --- a/cmd/reserve.go +++ b/cmd/reserve.go @@ -58,7 +58,7 @@ func (r reserve) Run(context wisski_distillery.Context) error { // check that the base directory does not exist logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase) - if fsx.IsDirectory(instance.FilesystemBase) { + if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) { return errProvisionAlreadyExists.WithMessageF(slug) } @@ -66,7 +66,7 @@ func (r reserve) Run(context wisski_distillery.Context) error { s := instance.Reserve() { if err := logging.LogOperation(func() error { - return s.Install(context.IOStream, component.InstallationContext{}) + return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{}) }, context.IOStream, "Installing docker stack"); err != nil { return err } diff --git a/cmd/snapshot.go b/cmd/snapshot.go index 5b24d0d..3d7f914 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -2,11 +2,11 @@ package cmd import ( "io/fs" - "os" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/core" "github.com/FAU-CDI/wisski-distillery/internal/wisski" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/targz" "github.com/tkw1536/goprogram/exit" @@ -60,7 +60,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { } defer func() { logging.LogMessage(context.IOStream, "Removing snapshot staging directory") - os.RemoveAll(sPath) + dis.Core.Environment.RemoveAll(sPath) }() } else { // staging mode: use dest as a destination @@ -74,8 +74,8 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { // create the directory (if it doesn't already exist) logging.LogMessage(context.IOStream, "Creating staging directory") - err = os.Mkdir(sPath, fs.ModePerm) - if !os.IsExist(err) && err != nil { + err = dis.Core.Environment.Mkdir(sPath, fs.ModePerm) + if !environment.IsExist(err) && err != nil { return errSnapshotFailed.WithMessageF(err) } err = nil @@ -92,7 +92,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { }) // write out the report, ignoring any errors! - sreport.WriteReport(context.IOStream) + sreport.WriteReport(dis.Core.Environment, context.IOStream) return nil }, context.IOStream, "Generating Snapshot") @@ -115,7 +115,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { if err := logging.LogOperation(func() error { context.IOStream.Println(archivePath) - count, err = targz.Package(archivePath, sPath, func(dst, src string) { + count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) { context.Printf("\033[2K\r%s", dst) }) context.Println("") diff --git a/cmd/system_update.go b/cmd/system_update.go index edfe032..c46c7e7 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -1,13 +1,12 @@ package cmd import ( - "os" - wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/internal/config" "github.com/FAU-CDI/wisski-distillery/internal/core" - "github.com/FAU-CDI/wisski-distillery/pkg/execx" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/unpack" "github.com/tkw1536/goprogram/exit" @@ -43,13 +42,10 @@ var errNoGraphDBZip = exit.Error{ } func (s systemupdate) AfterParse() error { - _, err := os.Stat(s.Positionals.GraphdbZip) - if os.IsNotExist(err) { + // TODO: Use a generic environment here! + if !fsx.IsFile(environment.Native{}, s.Positionals.GraphdbZip) { return errNoGraphDBZip.WithMessageF(s.Positionals.GraphdbZip) } - if err != nil { - return err - } return nil } @@ -85,7 +81,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { dis.SnapshotsArchivePath(), } { context.Println(d) - if err := os.MkdirAll(d, os.ModeDir); err != nil { + if err := dis.Core.Environment.MkdirAll(d, environment.DefaultDirPerm); err != nil { return errBoostrapFailedToCreateDirectory.WithMessageF(d, err) } } @@ -123,10 +119,10 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { if err := logging.LogOperation(func() error { for _, component := range dis.Installables() { - stack := component.Stack() + stack := component.Stack(dis.Core.Environment) ctx := component.Context(ctx) if err := logging.LogOperation(func() error { - return stack.Install(context.IOStream, ctx) + return stack.Install(dis.Core.Environment, context.IOStream, ctx) }, context.IOStream, "Installing docker stack %q", component.Name()); err != nil { return err } @@ -143,7 +139,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { } if err := logging.LogOperation(func() error { - return unpack.InstallDir(dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) { + return unpack.InstallDir(dis.Core.Environment, dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) { context.Printf("[copy] %s\n", dst) }) }, context.IOStream, "Unpacking Runtime Components"); err != nil { @@ -175,10 +171,11 @@ var errMustExecFailed = exit.Error{ // mustExec indicates that the given executable process must complete successfully. // If it does not, returns errMustExecFailed func (si systemupdate) mustExec(context wisski_distillery.Context, workdir string, exe string, argv ...string) error { + dis := context.Environment if workdir == "" { workdir = context.Environment.Config.DeployRoot } - code := execx.Exec(context.IOStream, workdir, exe, argv...) + code := dis.Core.Environment.Exec(context.IOStream, workdir, exe, argv...) if code != 0 { err := errMustExecFailed.WithMessageF(code) err.ExitCode = exit.ExitCode(code) diff --git a/cmd/update_prefix_config.go b/cmd/update_prefix_config.go index 8a7dcb0..3f60d44 100644 --- a/cmd/update_prefix_config.go +++ b/cmd/update_prefix_config.go @@ -1,11 +1,11 @@ package cmd import ( - "io/fs" - "os" + "io" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/tkw1536/goprogram/exit" ) @@ -42,7 +42,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error { target := ddis.ResolverConfigPath() // print the configuration - config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm) + config, err := dis.Core.Environment.Create(target, environment.DefaultFilePerm) if err != nil { return errPrefixUpdateFailed.WithMessageF(err) } @@ -58,7 +58,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error { context.IOStream.Printf("%s", data) // and write it out! - if _, err := config.WriteString(data); err != nil { + if _, err := io.WriteString(config, data); err != nil { return err } @@ -70,7 +70,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error { // and restart the resolver to apply the config! logging.LogMessage(context.IOStream, "restarting resolver stack") - if err := ddis.Stack().Restart(context.IOStream); err != nil { + if err := ddis.Stack(ddis.Environment).Restart(context.IOStream); err != nil { return errPrefixUpdateFailed.WithMessageF(err) } diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 544e56c..610bb72 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -2,14 +2,13 @@ package backup import ( - "io/fs" - "os" "path/filepath" "sync" "time" "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/internal/wisski" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/tkw1536/goprogram/stream" "golang.org/x/exp/slices" @@ -61,6 +60,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) { err: bc.Backup(context), } }(bc, &context{ + env: dis.Core.Environment, io: io, dst: filepath.Join(backup.Description.Dest, bc.BackupName()), files: files, @@ -73,7 +73,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) { defer wg.Done() instancesBackupDir := filepath.Join(backup.Description.Dest, "instances") - if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil { + if err := dis.Core.Environment.Mkdir(instancesBackupDir, environment.DefaultDirPerm); err != nil { backup.InstanceListErr = err return } @@ -89,7 +89,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) { for i, instance := range instances { backup.InstanceSnapshots[i] = func() wisski.Snapshot { dir := filepath.Join(instancesBackupDir, instance.Slug) - if err := os.Mkdir(dir, fs.ModeDir); err != nil { + if err := dis.Core.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil { return wisski.Snapshot{ ErrPanic: err, } diff --git a/internal/backup/context.go b/internal/backup/context.go index 25c4615..ec0d700 100644 --- a/internal/backup/context.go +++ b/internal/backup/context.go @@ -3,16 +3,16 @@ package backup import ( "errors" "io" - "io/fs" - "os" "path/filepath" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/tkw1536/goprogram/stream" ) // context implements [components.BackupContext] type context struct { + env environment.Environment io stream.IOStream dst string // destination directory files chan string // files channel @@ -54,7 +54,7 @@ func (bc *context) AddDirectory(path string, op func() error) error { } // run the make directory - if err := os.Mkdir(dst, fs.ModeDir); err != nil { + if err := bc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil { return err } @@ -72,7 +72,7 @@ func (bc *context) CopyFile(dst, src string) error { return err } bc.sendPath(dst) - return fsx.CopyFile(dstPath, src) + return fsx.CopyFile(bc.env, dstPath, src) } func (bc *context) AddFile(path string, op func(file io.Writer) error) error { @@ -83,7 +83,7 @@ func (bc *context) AddFile(path string, op func(file io.Writer) error) error { } // create the file - file, err := os.Create(dst) + file, err := bc.env.Create(dst, environment.DefaultFilePerm) if err != nil { return err } diff --git a/internal/backup/report.go b/internal/backup/report.go index a8ab6ec..de4180a 100644 --- a/internal/backup/report.go +++ b/internal/backup/report.go @@ -4,13 +4,13 @@ import ( "encoding/json" "fmt" "io" - "os" "path/filepath" "strings" "time" "github.com/FAU-CDI/wisski-distillery/internal/wisski" "github.com/FAU-CDI/wisski-distillery/pkg/countwriter" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/tkw1536/goprogram/stream" ) @@ -96,20 +96,20 @@ func (backup Backup) Report(w io.Writer) (int, error) { // WriteReport writes out the report belonging to this backup. // It is a separate function, to allow writing it indepenently of the rest. -func (backup Backup) WriteReport(io stream.IOStream) error { +func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error { return logging.LogOperation(func() error { reportPath := filepath.Join(backup.Description.Dest, "report.txt") - io.Println(reportPath) + stream.Println(reportPath) // create the report file! - report, err := os.Create(reportPath) + report, err := env.Create(reportPath, environment.DefaultFilePerm) if err != nil { return err } defer report.Close() // print the report into it! - _, err = report.WriteString(backup.String()) + _, err = io.WriteString(report, backup.String()) return err - }, io, "Writing backup report") + }, stream, "Writing backup report") } diff --git a/internal/component/backup.go b/internal/component/backup.go index 96f5a4f..027303d 100644 --- a/internal/component/backup.go +++ b/internal/component/backup.go @@ -28,8 +28,8 @@ type BackupContext interface { // It then allows op to fill the file. AddDirectory(path string, op func() error) error - // CopyFile copies a file from source to dst. - CopyFile(dest, src string) error + // CopyFile copies a file from src to dst. + CopyFile(dst, src string) error // AddFile creates a new file at the provided path inside the destination. // Passing the empty path creates the destination as a file. diff --git a/internal/component/component.go b/internal/component/component.go index 337e493..0da5a49 100644 --- a/internal/component/component.go +++ b/internal/component/component.go @@ -3,6 +3,7 @@ package component import ( "github.com/FAU-CDI/wisski-distillery/internal/config" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Component represents a logical subsystem of the distillery. @@ -38,7 +39,7 @@ type Installable interface { // Stack can be used to gain access to the "docker compose" stack. // // This should internally call [ComponentBase.MakeStack] - Stack() StackWithResources + Stack(env environment.Environment) StackWithResources // Context returns a new InstallationContext to be used during installation from the command line. // Typically this should just pass through the parent, but might perform other tasks. @@ -47,8 +48,14 @@ type Installable interface { // ComponentBase implements base functionality for a component type ComponentBase struct { - Dir string // Dir is the directory this component lives in - Config *config.Config // Config is the configuration of the underlying distillery + Core // the core of the associated distillery + Dir string // Dir is the directory this component lives in +} + +// Core represents the core of a distillery +type Core struct { + Environment environment.Environment // environment to use for reading / writing to and from the distillery + Config *config.Config // the configuration of the distillery } // Base returns a reference to the ComponentBase @@ -67,7 +74,8 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext { } // MakeStack registers the Installable as a stack -func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources { +func (cb ComponentBase) MakeStack(env environment.Environment, stack StackWithResources) StackWithResources { + stack.Env = env stack.Dir = cb.Dir return stack } diff --git a/internal/component/control/control.go b/internal/component/control/control.go index 6acb9e0..b39f5cd 100644 --- a/internal/component/control/control.go +++ b/internal/component/control/control.go @@ -6,6 +6,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/internal/component/instances" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Control represents the control server @@ -24,8 +25,8 @@ func (control Control) Name() string { //go:embed all:control control.env var resources embed.FS -func (control Control) Stack() component.StackWithResources { - return control.ComponentBase.MakeStack(component.StackWithResources{ +func (control Control) Stack(env environment.Environment) component.StackWithResources { + return control.ComponentBase.MakeStack(env, component.StackWithResources{ Resources: resources, ContextPath: "control", EnvPath: "control.env", @@ -49,6 +50,6 @@ func (control Control) Stack() component.StackWithResources { func (control Control) Context(parent component.InstallationContext) component.InstallationContext { return component.InstallationContext{ - core.Executable: control.Config.CurrentExecutable(), + core.Executable: control.Config.CurrentExecutable(control.Environment), // TODO: Does this make sense? } } diff --git a/internal/component/control/resolver.go b/internal/component/control/resolver.go index d43258d..d16e311 100644 --- a/internal/component/control/resolver.go +++ b/internal/component/control/resolver.go @@ -2,7 +2,6 @@ package control import ( "fmt" - "os" "path/filepath" "regexp" @@ -37,7 +36,7 @@ func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, // open the prefix file prefixFile := control.ResolverConfigPath() - fs, err := os.Open(prefixFile) + fs, err := control.Environment.Open(prefixFile) io.Println("loading prefixes from ", prefixFile) if err != nil { return p, err diff --git a/internal/component/control/self.go b/internal/component/control/self.go index 9d0ea8a..007c0c0 100644 --- a/internal/component/control/self.go +++ b/internal/component/control/self.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "os" "strings" "github.com/tkw1536/goprogram/stream" @@ -13,7 +12,7 @@ import ( // self returns the handler for the self overrides func (control Control) self(io stream.IOStream) (redirect Redirect, err error) { // open the overrides file - overrides, err := os.Open(control.Config.SelfOverridesFile) + overrides, err := control.Environment.Open(control.Config.SelfOverridesFile) io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile) if err != nil { return redirect, err diff --git a/internal/component/installable.go b/internal/component/installable.go index f3d04a7..4cea911 100644 --- a/internal/component/installable.go +++ b/internal/component/installable.go @@ -2,9 +2,9 @@ package component import ( "io/fs" - "os" "path/filepath" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/unpack" "github.com/pkg/errors" @@ -29,7 +29,7 @@ type StackWithResources struct { CopyContextFiles []string // Files to copy from the installation context - MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir + MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate] MakeDirs []string // directories to ensure that exist TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs @@ -42,10 +42,11 @@ type InstallationContext map[string]string // // Installation is non-interactive, but will provide debugging output onto io. // InstallationContext -func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error { +func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error { if is.ContextPath != "" { // setup the base files if err := unpack.InstallDir( + env, is.Dir, is.ContextPath, is.Resources, @@ -62,6 +63,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon if is.EnvPath != "" && is.EnvContext != nil { io.Printf("[config] %s\n", envDest) if err := unpack.InstallTemplate( + env, envDest, is.EnvContext, is.EnvPath, @@ -78,9 +80,9 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon io.Printf("[make] %s\n", dst) if is.MakeDirsPerm == fs.FileMode(0) { - is.MakeDirsPerm = fs.ModeDir + is.MakeDirsPerm = environment.DefaultDirPerm } - if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil { + if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil { return err } } @@ -98,7 +100,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon // copy over file from context io.Printf("[copy] %s (from %s)\n", dst, src) - if err := fsx.CopyFile(dst, src); err != nil { + if err := fsx.CopyFile(env, dst, src); err != nil { return errors.Wrapf(err, "Unable to copy file %s", src) } } @@ -109,7 +111,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon dst := filepath.Join(is.Dir, name) io.Printf("[touch] %s\n", dst) - if err := fsx.Touch(dst); err != nil { + if err := fsx.Touch(env, dst); err != nil { return err } } diff --git a/internal/component/instances/wisski_create.go b/internal/component/instances/wisski_create.go index ab34fd6..e8f8ff7 100644 --- a/internal/component/instances/wisski_create.go +++ b/internal/component/instances/wisski_create.go @@ -21,7 +21,7 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) { wisski.instances = instances // make sure that the slug is valid! - slug, err = stringparser.ParseSlug(slug) + slug, err = stringparser.ParseSlug(instances.Environment, slug) if err != nil { return wisski, errInvalidSlug } @@ -70,7 +70,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error { // create the basic st! st := wisski.Barrel() - if err := st.Install(io, component.InstallationContext{}); err != nil { + if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil { return err } diff --git a/internal/component/instances/wisski_pathbuilders.go b/internal/component/instances/wisski_pathbuilders.go index 7cf7d8b..4cbbcd1 100644 --- a/internal/component/instances/wisski_pathbuilders.go +++ b/internal/component/instances/wisski_pathbuilders.go @@ -2,12 +2,11 @@ package instances import ( "fmt" - "io/fs" - "os" "path/filepath" _ "embed" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/goprogram/stream" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -51,7 +50,7 @@ func (wisski *WissKI) ExportPathbuilders(dest string) error { for _, name := range names { pbxml := []byte(pathbuilders[name]) name := filepath.Join(dest, fmt.Sprintf("%s.xml", name)) - if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil { + if err := environment.WriteFile(wisski.instances.Core.Environment, name, pbxml, environment.DefaultFilePerm); err != nil { return err } } diff --git a/internal/component/instances/wisski_prefix.go b/internal/component/instances/wisski_prefix.go index ebaef07..df255a9 100644 --- a/internal/component/instances/wisski_prefix.go +++ b/internal/component/instances/wisski_prefix.go @@ -3,7 +3,6 @@ package instances import ( "errors" "io" - "os" "path/filepath" "strings" @@ -14,7 +13,7 @@ import ( // NoPrefix checks if this WissKI instance is excluded from generating prefixes. // TODO: Move this to the database! func (wisski *WissKI) NoPrefix() bool { - return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip")) + return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip")) } var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") @@ -41,8 +40,8 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) { // custom prefixes prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes") - if fsx.IsFile(prefixPath) { - prefix, err := os.Open(prefixPath) + if fsx.IsFile(wisski.instances.Environment, prefixPath) { + prefix, err := wisski.instances.Core.Environment.Open(prefixPath) if err != nil { return "", err } diff --git a/internal/component/instances/wisski_stack.go b/internal/component/instances/wisski_stack.go index 51ad924..461c68b 100644 --- a/internal/component/instances/wisski_stack.go +++ b/internal/component/instances/wisski_stack.go @@ -2,7 +2,6 @@ package instances import ( "embed" - "io/fs" "path/filepath" "github.com/FAU-CDI/wisski-distillery/internal/component" @@ -16,6 +15,7 @@ func (wisski WissKI) Barrel() component.StackWithResources { return component.StackWithResources{ Stack: component.Stack{ Dir: wisski.FilesystemBase, + Env: wisski.instances.Environment, }, Resources: barrelResources, @@ -35,8 +35,7 @@ func (wisski WissKI) Barrel() component.StackWithResources { "GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile, }, - MakeDirsPerm: fs.ModeDir | fs.ModePerm, - MakeDirs: []string{"data", ".composer"}, + MakeDirs: []string{"data", ".composer"}, TouchFiles: []string{ filepath.Join("data", "authorized_keys"), @@ -52,6 +51,7 @@ func (wisski WissKI) Reserve() component.StackWithResources { return component.StackWithResources{ Stack: component.Stack{ Dir: wisski.FilesystemBase, + Env: wisski.instances.Environment, }, Resources: reserveResources, diff --git a/internal/component/sql/backup.go b/internal/component/sql/backup.go index b78b729..b50ed1e 100644 --- a/internal/component/sql/backup.go +++ b/internal/component/sql/backup.go @@ -17,7 +17,7 @@ func (*SQL) BackupName() string { func (sql *SQL) Backup(context component.BackupContext) error { return context.AddFile("", func(file io.Writer) error { io := context.IO().Streams(file, nil, nil, 0).NonInteractive() - code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases") + code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--all-databases") if err != nil { return err } diff --git a/internal/component/sql/database.go b/internal/component/sql/database.go index b003606..986450e 100644 --- a/internal/component/sql/database.go +++ b/internal/component/sql/database.go @@ -21,6 +21,7 @@ func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, err DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database), DefaultStringSize: 256, } + // TODO: Use sql.Core.Environment.Dial db, err := gorm.Open(mysql.New(cfg), config) if err != nil { @@ -63,7 +64,7 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) { func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error { io = io.Streams(dest, nil, nil, 0).NonInteractive() - code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database) + code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database) if err != nil { return err } @@ -75,7 +76,7 @@ func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) err // OpenShell executes a mysql shell command func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) { - return sql.Stack().Exec(io, "sql", "mysql", argv...) + return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...) } // WaitShell waits for the sql database to be reachable via a docker-compose shell diff --git a/internal/component/sql/sql.go b/internal/component/sql/sql.go index 03bfd08..038f26f 100644 --- a/internal/component/sql/sql.go +++ b/internal/component/sql/sql.go @@ -3,10 +3,10 @@ package sql import ( "context" "embed" - "io/fs" "time" "github.com/FAU-CDI/wisski-distillery/internal/component" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) type SQL struct { @@ -25,12 +25,12 @@ func (SQL) Name() string { //go:embed all:sql var resources embed.FS -func (ssh SQL) Stack() component.StackWithResources { - return ssh.ComponentBase.MakeStack(component.StackWithResources{ +func (ssh SQL) Stack(env environment.Environment) component.StackWithResources { + return ssh.ComponentBase.MakeStack(env, component.StackWithResources{ Resources: resources, ContextPath: "sql", - MakeDirsPerm: fs.ModeDir | fs.ModePerm, + MakeDirsPerm: environment.DefaultDirPerm, MakeDirs: []string{ "data", }, diff --git a/internal/component/ssh/ssh.go b/internal/component/ssh/ssh.go index a19759f..9edbb27 100644 --- a/internal/component/ssh/ssh.go +++ b/internal/component/ssh/ssh.go @@ -4,6 +4,7 @@ import ( "embed" "github.com/FAU-CDI/wisski-distillery/internal/component" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) type SSH struct { @@ -17,8 +18,8 @@ func (SSH) Name() string { //go:embed all:stack var resources embed.FS -func (ssh SSH) Stack() component.StackWithResources { - return ssh.ComponentBase.MakeStack(component.StackWithResources{ +func (ssh SSH) Stack(env environment.Environment) component.StackWithResources { + return ssh.ComponentBase.MakeStack(env, component.StackWithResources{ Resources: resources, ContextPath: "stack", }) diff --git a/internal/component/stack.go b/internal/component/stack.go index c0a943d..a832f0a 100644 --- a/internal/component/stack.go +++ b/internal/component/stack.go @@ -6,7 +6,7 @@ import ( "bytes" "errors" - "github.com/FAU-CDI/wisski-distillery/pkg/execx" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/goprogram/stream" ) @@ -18,6 +18,7 @@ import ( type Stack struct { Dir string // Directory this Stack is located in + Env environment.Environment DockerExecutable string // Path to the native docker executable to use } @@ -101,7 +102,7 @@ func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string code, err := ds.compose(io, compose...) if err != nil { - return execx.ExecCommandError, nil + return environment.ExecCommandError, nil } return code, nil } @@ -175,10 +176,10 @@ func (ds Stack) Down(io stream.IOStream) error { func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) { if ds.DockerExecutable == "" { var err error - ds.DockerExecutable, err = execx.LookPathAbs("docker") + ds.DockerExecutable, err = ds.Env.LookPathAbs("docker") if err != nil { - return execx.ExecCommandError, err + return environment.ExecCommandError, err } } - return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil + return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil } diff --git a/internal/component/triplestore/database.go b/internal/component/triplestore/database.go index dd6c6c6..3e2f132 100644 --- a/internal/component/triplestore/database.go +++ b/internal/component/triplestore/database.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "mime/multipart" + "net" "net/http" "github.com/FAU-CDI/wisski-distillery/pkg/logging" @@ -61,6 +62,14 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str } // create the request object + client := &http.Client{ + Transport: &http.Transport{ + Dial: ts.Environment.Dial, + DialTLS: func(network, addr string) (net.Conn, error) { + return nil, errors.New("not implemented") + }, + }, + } req, err := http.NewRequest(method, ts.BaseURL+url, reader) if err != nil { return nil, err @@ -76,7 +85,7 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword) // and send it - return http.DefaultClient.Do(req) + return client.Do(req) } // Wait waits for the connection to the Triplestore to succeed. diff --git a/internal/component/triplestore/triplestore.go b/internal/component/triplestore/triplestore.go index c20d621..023ecfd 100644 --- a/internal/component/triplestore/triplestore.go +++ b/internal/component/triplestore/triplestore.go @@ -3,11 +3,11 @@ package triplestore import ( "context" "embed" - "io/fs" "path/filepath" "time" "github.com/FAU-CDI/wisski-distillery/internal/component" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) type Triplestore struct { @@ -26,14 +26,13 @@ func (Triplestore) Name() string { //go:embed all:stack var resources embed.FS -func (ts Triplestore) Stack() component.StackWithResources { - return ts.ComponentBase.MakeStack(component.StackWithResources{ +func (ts Triplestore) Stack(env environment.Environment) component.StackWithResources { + return ts.ComponentBase.MakeStack(env, component.StackWithResources{ Resources: resources, ContextPath: "stack", CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant? - MakeDirsPerm: fs.ModeDir | fs.ModePerm, MakeDirs: []string{ filepath.Join("data", "data"), filepath.Join("data", "work"), diff --git a/internal/component/web/web.go b/internal/component/web/web.go index ce011d8..f9fcdb7 100644 --- a/internal/component/web/web.go +++ b/internal/component/web/web.go @@ -4,6 +4,7 @@ import ( "embed" "github.com/FAU-CDI/wisski-distillery/internal/component" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Web implements the ingress gateway for the distillery. @@ -17,11 +18,11 @@ func (Web) Name() string { return "web" } -func (web Web) Stack() component.StackWithResources { +func (web Web) Stack(env environment.Environment) component.StackWithResources { if web.Config.HTTPSEnabled() { - return web.stackHTTPS() + return web.stackHTTPS(env) } else { - return web.stackHTTP() + return web.stackHTTP(env) } } @@ -29,8 +30,8 @@ func (web Web) Stack() component.StackWithResources { //go:embed web-https.env var httpsResources embed.FS -func (web Web) stackHTTPS() component.StackWithResources { - return web.MakeStack(component.StackWithResources{ +func (web Web) stackHTTPS(env environment.Environment) component.StackWithResources { + return web.MakeStack(env, component.StackWithResources{ Resources: httpsResources, ContextPath: "web-https", EnvPath: "web-https.env", @@ -45,8 +46,8 @@ func (web Web) stackHTTPS() component.StackWithResources { //go:embed web-http.env var httpResources embed.FS -func (web Web) stackHTTP() component.StackWithResources { - return web.MakeStack(component.StackWithResources{ +func (web Web) stackHTTP(env environment.Environment) component.StackWithResources { + return web.MakeStack(env, component.StackWithResources{ Resources: httpResources, ContextPath: "web-http", EnvPath: "web-http.env", diff --git a/internal/config/executable.go b/internal/config/executable.go index 2f0bd50..957c2bb 100644 --- a/internal/config/executable.go +++ b/internal/config/executable.go @@ -1,10 +1,10 @@ package config import ( - "os" "path/filepath" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" ) @@ -14,19 +14,19 @@ func (cfg Config) ExecutablePath() string { } // UsingDistilleryExecutable checks if the current process is using the distillery executable -func (cfg Config) UsingDistilleryExecutable() bool { - exe, err := os.Executable() +func (cfg Config) UsingDistilleryExecutable(env environment.Environment) bool { + exe, err := env.Executable() if err != nil { return false } - return fsx.SameFile(exe, cfg.ExecutablePath()) + return fsx.SameFile(env, exe, cfg.ExecutablePath()) } // CurrentExecutable returns the path to the current executable being used. // When it does not exist, falls back to the default executable. -func (cfg Config) CurrentExecutable() string { - exe, err := os.Executable() - if err != nil || !fsx.IsFile(exe) { +func (cfg Config) CurrentExecutable(env environment.Environment) string { + exe, err := env.Executable() + if err != nil || !fsx.IsFile(env, exe) { return cfg.ExecutablePath() } return exe diff --git a/internal/config/read.go b/internal/config/read.go index 66ff1fa..bddb774 100644 --- a/internal/config/read.go +++ b/internal/config/read.go @@ -4,6 +4,7 @@ import ( "io" "reflect" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/envreader" "github.com/FAU-CDI/wisski-distillery/pkg/stringparser" ) @@ -16,7 +17,7 @@ import ( // When a key is missing, it is set to the default value. // // See also [stringparser.Parse]. -func (config *Config) Unmarshal(src io.Reader) error { +func (config *Config) Unmarshal(env environment.Environment, src io.Reader) error { // read all the values! values, err := envreader.ReadAll(src) if err != nil { @@ -32,26 +33,26 @@ func (config *Config) Unmarshal(src io.Reader) error { tField := tConfig.Field(i) vField := vConfig.FieldByName(tField.Name) - env := tField.Tag.Get("env") - dflt := tField.Tag.Get("default") - parser := tField.Tag.Get("parser") + tEnv := tField.Tag.Get("env") + tDefault := tField.Tag.Get("default") + tParser := tField.Tag.Get("parser") // skip it if it isn't loaded! - if env == "" { + if tEnv == "" { continue } // read the value with a default - value, ok := values[env] + value, ok := values[tEnv] if !ok || value == "" { - if dflt == "" { + if tDefault == "" { continue } - value = dflt + value = tDefault } // parse the value! - if err := stringparser.Parse(parser, value, vField); err != nil { + if err := stringparser.Parse(env, tParser, value, vField); err != nil { return err } } diff --git a/internal/config/template.go b/internal/config/template.go index 2195c90..3517149 100644 --- a/internal/config/template.go +++ b/internal/config/template.go @@ -7,6 +7,7 @@ import ( "reflect" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/hostname" "github.com/FAU-CDI/wisski-distillery/pkg/password" "github.com/FAU-CDI/wisski-distillery/pkg/unpack" @@ -29,13 +30,13 @@ type Template struct { } // SetDefaults sets defaults on the template -func (tpl *Template) SetDefaults() (err error) { +func (tpl *Template) SetDefaults(env environment.Environment) (err error) { if tpl.DeployRoot == "" { tpl.DeployRoot = core.BaseDirectoryDefault } if tpl.DefaultDomain == "" { - tpl.DefaultDomain = hostname.FQDN() + tpl.DefaultDomain = hostname.FQDN(env) } if tpl.SelfOverridesFile == "" { diff --git a/internal/core/meta.go b/internal/core/meta.go index 2d446ff..ac6aca5 100644 --- a/internal/core/meta.go +++ b/internal/core/meta.go @@ -3,10 +3,11 @@ package core import ( "errors" "io/fs" - "os" "os/user" "path/filepath" "strings" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // MetaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable. @@ -33,7 +34,7 @@ var errReadBaseDirectoryEmpty = errors.New("ReadBaseDirectory: Directory is empt // Use [ParamsFromEnv] to initialize parameters completely. // // It does not perform any reading of files. -func ReadBaseDirectory() (value string, err error) { +func ReadBaseDirectory(env environment.Environment) (value string, err error) { // get the path! path, err := MetaConfigPath() if err != nil { @@ -41,7 +42,7 @@ func ReadBaseDirectory() (value string, err error) { } // read the meta config file! - contents, err := os.ReadFile(path) + contents, err := environment.ReadFile(env, path) if err != nil { return "", err } @@ -59,7 +60,7 @@ func ReadBaseDirectory() (value string, err error) { } // WriteBaseDirectory writes the base directory to the environment, or returns an error -func WriteBaseDirectory(dir string) error { +func WriteBaseDirectory(env environment.Environment, dir string) error { // get the path! path, err := MetaConfigPath() if err != nil { @@ -67,5 +68,5 @@ func WriteBaseDirectory(dir string) error { } // just put the directory inside it! - return os.WriteFile(path, []byte(dir), fs.ModePerm) + return environment.WriteFile(env, path, []byte(dir), fs.ModePerm) } diff --git a/internal/core/params.go b/internal/core/params.go index 008a0b9..cc7cabd 100644 --- a/internal/core/params.go +++ b/internal/core/params.go @@ -1,8 +1,9 @@ package core import ( - "os" "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Params are used to initialize the excutable. @@ -15,9 +16,9 @@ type Params struct { func ParamsFromEnv() (params Params, err error) { // try to read the base directory! - value, err := ReadBaseDirectory() + value, err := ReadBaseDirectory(environment.Native{}) // TODO: Are we sure about the native environment here? switch { - case os.IsNotExist(err): + case environment.IsNotExist(err): params.ConfigPath = BaseDirectoryDefault case err == nil: params.ConfigPath = value diff --git a/internal/wisski/backup_prune.go b/internal/wisski/backup_prune.go index c85fab7..3efd5aa 100644 --- a/internal/wisski/backup_prune.go +++ b/internal/wisski/backup_prune.go @@ -1,7 +1,6 @@ package wisski import ( - "os" "path/filepath" "time" @@ -18,7 +17,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error { sPath := dis.SnapshotsArchivePath() // list all the files - entries, err := os.ReadDir(sPath) + entries, err := dis.Core.Environment.ReadDir(sPath) if err != nil { return err } @@ -44,7 +43,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error { path := filepath.Join(sPath, entry.Name()) io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge) - if err := os.Remove(path); err != nil { + if err := dis.Core.Environment.Remove(path); err != nil { return err } } diff --git a/internal/wisski/component.go b/internal/wisski/component.go index 2629116..e05eb81 100644 --- a/internal/wisski/component.go +++ b/internal/wisski/component.go @@ -58,7 +58,7 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C], } base := c.Base() - base.Config = dis.Config + base.Core = dis.Core if base.Dir == "" { base.Dir = filepath.Join(dis.Config.DeployRoot, "core", c.Name()) } diff --git a/internal/wisski/distillery.go b/internal/wisski/distillery.go index 7ffc88d..e73ce30 100644 --- a/internal/wisski/distillery.go +++ b/internal/wisski/distillery.go @@ -3,16 +3,15 @@ package wisski import ( "context" - "github.com/FAU-CDI/wisski-distillery/internal/config" + "github.com/FAU-CDI/wisski-distillery/internal/component" ) // Distillery represents a WissKI Distillery // // It is the main structure used to interact with different components. type Distillery struct { - // Config holds the configuration of the distillery. - // It is read directly from a configuration file. - Config *config.Config + // core holds the core of the distillery + component.Core // Upstream holds information to connect to the various running // distillery components. @@ -25,7 +24,7 @@ type Distillery struct { components } -// Upstream are the upstream urls connecting to the various external components. +// Upstream contains the configuration for accessing remote configuration. type Upstream struct { SQL string Triplestore string diff --git a/internal/wisski/init.go b/internal/wisski/init.go index b5c3091..d747ae3 100644 --- a/internal/wisski/init.go +++ b/internal/wisski/init.go @@ -1,10 +1,10 @@ package wisski import ( - "os" - + "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/internal/config" "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/tkw1536/goprogram/exit" ) @@ -21,6 +21,9 @@ var errOpenConfig = exit.Error{ // NewDistillery creates a new distillery from the provided flags func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) { dis = &Distillery{ + Core: component.Core{ + Environment: environment.Native{}, + }, Upstream: Upstream{ SQL: "127.0.0.1:3306", Triplestore: "127.0.0.1:7200", @@ -34,7 +37,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) if flags.InternalInDocker { dis.Upstream.SQL = "sql:3306" dis.Upstream.Triplestore = "triplestore:7200" - params.ConfigPath = os.Getenv("CONFIG_PATH") + params.ConfigPath = dis.Core.Environment.GetEnv("CONFIG_PATH") } // if we don't need to load the config, there is nothing to do @@ -52,7 +55,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) } // open the config file! - f, err := os.Open(params.ConfigPath) + f, err := dis.Core.Environment.Open(params.ConfigPath) if err != nil { return nil, errOpenConfig.WithMessageF(err) } @@ -62,6 +65,6 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) dis.Config = &config.Config{ ConfigPath: cfg, } - err = dis.Config.Unmarshal(f) + err = dis.Config.Unmarshal(dis.Core.Environment, f) return } diff --git a/internal/wisski/snapshot.go b/internal/wisski/snapshot.go index b9c4d60..43ac304 100644 --- a/internal/wisski/snapshot.go +++ b/internal/wisski/snapshot.go @@ -4,8 +4,6 @@ import ( "encoding/json" "fmt" "io" - "io/fs" - "os" "path/filepath" "strings" "time" @@ -13,6 +11,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" "github.com/FAU-CDI/wisski-distillery/internal/component/instances" "github.com/FAU-CDI/wisski-distillery/pkg/countwriter" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/opgroup" @@ -42,7 +41,7 @@ func (dis *Distillery) SnapshotsArchivePath() string { // The path is guaranteed to not exist. func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) { // TODO: Consider moving these into a subdirectory with the provided prefix. - for path == "" || fsx.Exists(path) { + for path == "" || fsx.Exists(dis.Environment, path) { name := dis.newSnapshotName(prefix) + ".tar.gz" path = filepath.Join(dis.SnapshotsArchivePath(), name) } @@ -64,9 +63,9 @@ func (*Distillery) newSnapshotName(prefix string) string { // NewSnapshotStagingDir returns the path to a new snapshot directory. // The directory is guaranteed to have been freshly created. func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) { - for path == "" || os.IsExist(err) { + for path == "" || environment.IsExist(err) { path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix)) - err = os.Mkdir(path, os.ModeDir) + err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm) } if err != nil { path = "" @@ -210,7 +209,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt") files <- bkPath - info, err := os.Create(bkPath) + info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm) if err != nil { return err } @@ -227,7 +226,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase)) // copy over whatever is in the base directory - return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { + return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) { files <- dst }) }, &snapshot.ErrFilesystem) @@ -237,7 +236,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq") files <- tsPath - nquads, err := os.Create(tsPath) + nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm) if err != nil { return err } @@ -253,7 +252,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql") files <- sqlPath - sql, err := os.Create(sqlPath) + sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm) if err != nil { return err } @@ -279,7 +278,7 @@ func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, inst files <- pbPath // create the directory! - if err := os.Mkdir(pbPath, fs.ModeDir); err != nil { + if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil { return err } @@ -312,20 +311,20 @@ func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[stri // WriteReport writes out the report belonging to this snapshot. // It is a separate function, to allow writing it indepenently of the rest. -func (snapshot Snapshot) WriteReport(io stream.IOStream) error { +func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error { return logging.LogOperation(func() error { reportPath := filepath.Join(snapshot.Description.Dest, "report.txt") - io.Println(reportPath) + stream.Println(reportPath) // create the report file! - report, err := os.Create(reportPath) + report, err := env.Create(reportPath, environment.DefaultFilePerm) if err != nil { return err } defer report.Close() // print the report into it! - _, err = report.WriteString(snapshot.String()) + _, err = io.WriteString(report, snapshot.String()) return err - }, io, "Writing snapshot report") + }, stream, "Writing snapshot report") } diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go new file mode 100644 index 0000000..1725de9 --- /dev/null +++ b/pkg/environment/environment.go @@ -0,0 +1,58 @@ +package environment + +import ( + "io" + "io/fs" + "net" + "time" + + "github.com/tkw1536/goprogram/stream" +) + +// Environment represents an environment that a program can run it. +// It mostly mimics the interfaces of the [os] package. +type Environment interface { + isEnv() + + GetEnv(name string) string + + Stat(path string) (fs.FileInfo, error) + Lstat(path string) (fs.FileInfo, error) + + Readlink(path string) (string, error) + Symlink(oldname, newname string) error + + ReadDir(name string) ([]fs.DirEntry, error) + + Open(path string) (fs.File, error) + Chtimes(name string, atime time.Time, mtime time.Time) error + SameFile(f1, f2 fs.FileInfo) bool + + Create(path string, mode fs.FileMode) (WritableFile, error) + + Mkdir(path string, mode fs.FileMode) error + MkdirAll(path string, mode fs.FileMode) error + + Remove(path string) error + RemoveAll(path string) error + + WalkDir(root string, fn fs.WalkDirFunc) error + + Abs(path string) (string, error) + + Listen(network, address string) (net.Listener, error) + Dial(network, address string) (net.Conn, error) + + Executable() (string, error) + Exec(io stream.IOStream, workdir string, exe string, argv ...string) int + LookPathAbs(name string) (string, error) +} + +type WritableFile interface { + fs.File + io.Writer +} + +func init() { + var _ Environment = Native{} +} diff --git a/pkg/environment/globals.go b/pkg/environment/globals.go new file mode 100644 index 0000000..8581d9c --- /dev/null +++ b/pkg/environment/globals.go @@ -0,0 +1,69 @@ +package environment + +import ( + "bytes" + "io" + "io/fs" + "os" + + "github.com/tkw1536/goprogram/stream" +) + +// ExecCommandError is returned by Exec when a command could not be executed. +// This typically hints that the executable cannot be found, but may have other causes. +const ExecCommandError = 127 + +// DefaultFilePerm is the default mode to use for files +const DefaultFilePerm fs.FileMode = 0666 + +// DefaultDirPerm is the default mode to use for directories +const DefaultDirPerm fs.FileMode = os.ModeDir & fs.ModePerm + +// IsExist checks if the provided error represents a 'does not exist' errror +func IsExist(err error) bool { + return os.IsExist(err) +} + +// IsNotExist checks if the provided error represents a 'does exist' error +func IsNotExist(err error) bool { + return os.IsNotExist(err) +} + +// WriteFile is like [os.WriteFile]. +func WriteFile(env Environment, path string, data []byte, mode fs.FileMode) error { + handle, err := env.Create(path, mode) + if err != nil { + return err + } + defer handle.Close() + + if _, err := handle.Write(data); err != nil { + return err + } + + return nil +} + +// ReadFile is like [os.ReadFile] +func ReadFile(env Environment, path string) ([]byte, error) { + // open the file! + file, err := env.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + // copy everything into a buffer! + var buffer bytes.Buffer + if _, err := io.Copy(&buffer, file); err != nil { + return nil, err + } + + // return the buffer contents! + return buffer.Bytes(), nil +} + +// MustExec is like Exec, except that it returns true if the command exited successfully, and else false. +func MustExec(env Environment, io stream.IOStream, workdir string, exe string, argv ...string) bool { + return env.Exec(io, workdir, exe, argv...) == 0 +} diff --git a/pkg/execx/exec.go b/pkg/environment/native_exec.go similarity index 53% rename from pkg/execx/exec.go rename to pkg/environment/native_exec.go index c9bb760..27f19a1 100644 --- a/pkg/execx/exec.go +++ b/pkg/environment/native_exec.go @@ -1,5 +1,4 @@ -// Package execx defines extensions to the "os/exec" package -package execx +package environment import ( "os/exec" @@ -7,15 +6,11 @@ import ( "github.com/tkw1536/goprogram/stream" ) -// ExecCommandError is returned by Exec when a command could not be executed. -// This typically hints that the executable cannot be found, but may have other causes. -const ExecCommandError = 127 - // Exec executes a system command with the specified input/output streams, working directory, and arguments. // // If the command executes, it's exit code will be returned. // If the command can not be executed, returns [ExecCommandError]. -func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { +func (Native) Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { // setup the command cmd := exec.Command(exe, argv...) cmd.Dir = workdir @@ -40,7 +35,10 @@ func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { return 0 } -// MustExec is like Exec, except that it returns true if the command exited successfully, and else false. -func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool { - return Exec(io, workdir, exe, argv...) == 0 +func (n Native) LookPathAbs(file string) (string, error) { + path, err := exec.LookPath(file) + if err != nil { + return "", err + } + return n.Abs(path) } diff --git a/pkg/environment/native_fs.go b/pkg/environment/native_fs.go new file mode 100644 index 0000000..dc132ea --- /dev/null +++ b/pkg/environment/native_fs.go @@ -0,0 +1,80 @@ +package environment + +import ( + "io/fs" + "os" + "path/filepath" + "time" +) + +type Native struct{} + +func (Native) isEnv() {} + +func (Native) GetEnv(name string) string { + return os.Getenv(name) +} + +func (Native) Stat(path string) (fs.FileInfo, error) { + return os.Stat(path) +} + +func (Native) Lstat(path string) (fs.FileInfo, error) { + return os.Lstat(path) +} + +func (Native) Readlink(path string) (string, error) { + return os.Readlink(path) +} + +func (Native) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} + +func (Native) ReadDir(name string) ([]fs.DirEntry, error) { + return os.ReadDir(name) +} + +func (Native) SameFile(f1, f2 fs.FileInfo) bool { + return os.SameFile(f1, f2) +} + +func (Native) WalkDir(path string, f fs.WalkDirFunc) error { + return filepath.WalkDir(path, f) +} + +func (Native) Executable() (string, error) { + return os.Executable() // TODO: not sure this works with the remote concepts +} + +func (Native) Open(path string) (fs.File, error) { + return os.Open(path) +} + +func (Native) Create(path string, mode fs.FileMode) (WritableFile, error) { + return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) +} + +func (Native) Chtimes(name string, atime time.Time, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +func (Native) Mkdir(path string, mode fs.FileMode) error { + return os.Mkdir(path, mode) +} + +func (Native) MkdirAll(path string, mode fs.FileMode) error { + return os.MkdirAll(path, mode) +} + +func (Native) Remove(path string) error { + return os.Remove(path) +} + +func (Native) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (Native) Abs(path string) (string, error) { + return filepath.Abs(path) +} diff --git a/pkg/environment/native_net.go b/pkg/environment/native_net.go new file mode 100644 index 0000000..2d47e1e --- /dev/null +++ b/pkg/environment/native_net.go @@ -0,0 +1,11 @@ +package environment + +import "net" + +func (Native) Listen(network, address string) (net.Listener, error) { + return net.Listen(network, address) +} + +func (Native) Dial(network, address string) (net.Conn, error) { + return net.Dial(network, address) +} diff --git a/pkg/execx/look.go b/pkg/execx/look.go deleted file mode 100644 index e816519..0000000 --- a/pkg/execx/look.go +++ /dev/null @@ -1,15 +0,0 @@ -package execx - -import ( - "os/exec" - "path/filepath" -) - -// LookPathAbs is like [exec.LookPath], but always returns an absolute path -func LookPathAbs(file string) (string, error) { - path, err := exec.LookPath(file) - if err != nil { - return "", err - } - return filepath.Abs(path) -} diff --git a/pkg/fsx/copy.go b/pkg/fsx/copy.go index 55e6045..9e69a8a 100644 --- a/pkg/fsx/copy.go +++ b/pkg/fsx/copy.go @@ -4,8 +4,9 @@ import ( "errors" "io" "io/fs" - "os" "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) var ErrCopySameFile = errors.New("src and dst must be different") @@ -14,13 +15,13 @@ var ErrCopySameFile = errors.New("src and dst must be different") // When src points to a symbolic link, will copy the symbolic link. // // When dst and src are the same file, returns ErrCopySameFile. -func CopyFile(dst, src string) error { - if SameFile(src, dst) { +func CopyFile(env environment.Environment, dst, src string) error { + if SameFile(env, src, dst) { return ErrCopySameFile } // open the source - srcFile, err := os.Open(src) + srcFile, err := env.Open(src) if err != nil { return err } @@ -33,7 +34,7 @@ func CopyFile(dst, src string) error { } // open or create the destination - dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode()) + dstFile, err := env.Create(dst, srcStat.Mode()) if err != nil { return err } @@ -46,27 +47,27 @@ func CopyFile(dst, src string) error { // CopyLink copies a link from src to dst. // If dst already exists, it is deleted and then re-created. -func CopyLink(dst, src string) error { +func CopyLink(env environment.Environment, dst, src string) error { // if they're the same file that is an error - if SameFile(dst, src) { + if SameFile(env, dst, src) { return ErrCopySameFile } // read the link target - target, err := os.Readlink(src) + target, err := env.Readlink(src) if err != nil { return err } // delete it if it already exists - if Exists(dst) { - if err := os.Remove(dst); err != nil { + if Exists(env, dst) { + if err := env.Remove(dst); err != nil { return err } } // make the symbolic link! - return os.Symlink(target, dst) + return env.Symlink(target, dst) } var ErrDstFile = errors.New("dst is a file") @@ -77,16 +78,16 @@ var ErrDstFile = errors.New("dst is a file") // When a directory already exists, additional files are not deleted. // // onCopy, when not nil, is called for each file or directory being copied. -func CopyDirectory(dst, src string, onCopy func(dst, src string)) error { +func CopyDirectory(env environment.Environment, dst, src string, onCopy func(dst, src string)) error { // sanity checks - if SameFile(src, dst) { + if SameFile(env, src, dst) { return ErrCopySameFile } - if IsFile(dst) { + if IsFile(env, dst) { return ErrDstFile } - return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + return env.WalkDir(src, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -111,19 +112,19 @@ func CopyDirectory(dst, src string, onCopy func(dst, src string)) error { } // if we have a symbolic link, copy the link! - if info.Mode()&os.ModeSymlink != 0 { - return CopyLink(dst, path) + if info.Mode()&fs.ModeSymlink != 0 { + return CopyLink(env, dst, path) } // if we got a file, we should copy it normally if !d.IsDir() { - return CopyFile(dst, path) + return CopyFile(env, dst, path) } // create the directory, but ignore an error if the directory already exists. // this is so that we can copy one tree into another tree. - err = os.Mkdir(dst, info.Mode()) - if os.IsExist(err) && IsDirectory(dst) { + err = env.Mkdir(dst, info.Mode()) + if environment.IsExist(err) && IsDirectory(env, dst) { err = nil } diff --git a/pkg/fsx/same.go b/pkg/fsx/same.go index 5414754..d083a95 100644 --- a/pkg/fsx/same.go +++ b/pkg/fsx/same.go @@ -1,17 +1,18 @@ package fsx import ( - "os" "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // SameFile checks if path1 and path2 refer to the same file. -// If both files exist, they are compared using [os.SameFile]. +// If both files exist, they are compared using [env.SameFile]. // If both files do not exist, the paths are first compared syntactically and then via recursion on [filepath.Dir]. -func SameFile(path1, path2 string) bool { +func SameFile(env environment.Environment, path1, path2 string) bool { // initial attempt: check if directly - same, certain := couldBeSameFile(path1, path2) + same, certain := couldBeSameFile(env, path1, path2) if certain { return same } @@ -28,28 +29,28 @@ func SameFile(path1, path2 string) bool { // compare the base names! { - same, _ := couldBeSameFile(d1, d2) + same, _ := couldBeSameFile(env, d1, d2) return same } } // couldBeSameFile checks if path1 might be the same as path2. // -// If both files exist, compares using [os.SameFile]. +// If both files exist, compares using [env.SameFile]. // Otherwise compares absolute paths using string comparison. // // same indicates if they might be the same file. // authorative indiciates if the result is authorative. -func couldBeSameFile(path1, path2 string) (same, authorative bool) { +func couldBeSameFile(env environment.Environment, path1, path2 string) (same, authorative bool) { { // stat both files - info1, err1 := os.Stat(path1) - info2, err2 := os.Stat(path2) + info1, err1 := env.Stat(path1) + info2, err2 := env.Stat(path2) - // both files exist => check using os.SameFile + // both files exist => check using env.SameFile // the result is always authorative if err1 == nil && err2 == nil { - same = os.SameFile(info1, info2) + same = env.SameFile(info1, info2) authorative = true return } @@ -60,15 +61,15 @@ func couldBeSameFile(path1, path2 string) (same, authorative bool) { } // only 1 file does not exist => they could be different - if os.IsNotExist(err1) != os.IsNotExist(err2) { + if environment.IsNotExist(err1) != environment.IsNotExist(err2) { return } } { // resolve paths absolutely - rpath1, err1 := filepath.Abs(path1) - rpath2, err2 := filepath.Abs(path2) + rpath1, err1 := env.Abs(path1) + rpath2, err2 := env.Abs(path2) // if either path could not be resolved absolutely // fallback to just using clean! diff --git a/pkg/fsx/touch.go b/pkg/fsx/touch.go index 4822536..aefa694 100644 --- a/pkg/fsx/touch.go +++ b/pkg/fsx/touch.go @@ -1,20 +1,21 @@ package fsx import ( - "os" "time" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Touch touches a file. // It is similar to the unix 'touch' command. // -// If the file does not exist exists, it is created using [os.Create]. +// If the file does not exist exists, it is created using [env.Create]. // If the file does exist, it's access and modification times are updated to the current time. -func Touch(path string) error { - _, err := os.Stat(path) +func Touch(env environment.Environment, path string) error { + _, err := env.Stat(path) switch { - case os.IsNotExist(err): - f, err := os.Create(path) + case environment.IsNotExist(err): + f, err := env.Create(path, environment.DefaultFilePerm) if err != nil { return err } @@ -24,6 +25,6 @@ func Touch(path string) error { return err default: now := time.Now().Local() - return os.Chtimes(path, now, now) + return env.Chtimes(path, now, now) } } diff --git a/pkg/fsx/type.go b/pkg/fsx/type.go index 64c4e5e..2e927ba 100644 --- a/pkg/fsx/type.go +++ b/pkg/fsx/type.go @@ -2,29 +2,31 @@ package fsx import ( - "os" + "io/fs" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Exists checks if the given path exists -func Exists(path string) bool { - _, err := os.Lstat(path) +func Exists(env environment.Environment, path string) bool { + _, err := env.Lstat(path) return err == nil } // IsDirectory checks if the provided path exists and is a directory -func IsDirectory(path string) bool { - info, err := os.Stat(path) +func IsDirectory(env environment.Environment, path string) bool { + info, err := env.Stat(path) return err == nil && info.Mode().IsDir() } // IsFile checks if the provided path exists and is a regular file -func IsFile(path string) bool { - info, err := os.Stat(path) +func IsFile(env environment.Environment, path string) bool { + info, err := env.Stat(path) return err == nil && info.Mode().IsRegular() } // IsLink checks if the provided path exists and is a symlink -func IsLink(path string) bool { - info, err := os.Lstat(path) - return err == nil && info.Mode()&os.ModeSymlink != 0 +func IsLink(env environment.Environment, path string) bool { + info, err := env.Lstat(path) + return err == nil && info.Mode()&fs.ModeSymlink != 0 } diff --git a/pkg/hostname/hostname.go b/pkg/hostname/hostname.go index a3ecea8..3984b3c 100644 --- a/pkg/hostname/hostname.go +++ b/pkg/hostname/hostname.go @@ -4,12 +4,14 @@ package hostname import ( "os" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/Showmax/go-fqdn" ) // FQDN attempts to return the fully qualified domain name of the host system. // If an error occurs, may fall back to the empty string. -func FQDN() string { +func FQDN(env environment.Environment) string { + // TODO: Pass this through! // try the hostname function { diff --git a/pkg/stringparser/parse.go b/pkg/stringparser/parse.go index 45e0945..a294548 100644 --- a/pkg/stringparser/parse.go +++ b/pkg/stringparser/parse.go @@ -4,11 +4,12 @@ import ( "reflect" "strings" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/pkg/errors" ) // Parse parses the provided value with the parser. -func Parse(name, value string, vField reflect.Value) error { +func Parse(env environment.Environment, name, value string, vField reflect.Value) error { // use the validator parser, ok := knownParsers[strings.ToLower(name)] @@ -17,7 +18,7 @@ func Parse(name, value string, vField reflect.Value) error { } // get the parsed value - checked, err := parser(value) + checked, err := parser(env, value) if err != nil { return errors.Wrapf(err, "parser %s returned error", name) } @@ -53,8 +54,8 @@ var knownParsers map[string]Parser[any] = map[string]Parser[any]{ } func asGenericParser[T any](parser Parser[T]) Parser[any] { - return func(s string) (value any, err error) { - value, err = parser(s) + return func(env environment.Environment, s string) (value any, err error) { + value, err = parser(env, s) return } } diff --git a/pkg/stringparser/stringparser.go b/pkg/stringparser/stringparser.go index bd56ce7..f18c98a 100644 --- a/pkg/stringparser/stringparser.go +++ b/pkg/stringparser/stringparser.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/pkg/errors" ) @@ -17,19 +18,19 @@ import ( // Parsers can be found in this package as functions called Parse*. // They are refered to by their name, e.g. ParseNonempty can be refered to by the name 'Nonempty'. // See [Parse]. -type Parser[T any] func(s string) (T, error) +type Parser[T any] func(env environment.Environment, s string) (T, error) // ParseAbspath checks that s is an absolute path and returns it as-is -func ParseAbspath(s string) (string, error) { - if !fsx.IsDirectory(s) { +func ParseAbspath(env environment.Environment, s string) (string, error) { + if !fsx.IsDirectory(env, s) { return "", errors.Errorf("%q does not exist or is not a directory", s) } return s, nil } // ParseFile checks that s is a valid file and returns it as-is -func ParseFile(s string) (string, error) { - if !fsx.IsFile(s) { +func ParseFile(env environment.Environment, s string) (string, error) { + if !fsx.IsFile(env, s) { return "", errors.Errorf("%q does not exist or is not a regular file", s) } return s, nil @@ -38,7 +39,7 @@ func ParseFile(s string) (string, error) { var errEmptyString = errors.New("value is empty") // ParseNonEmpty checks that s is a non-empty string and returns it as-is -func ParseNonEmpty(s string) (string, error) { +func ParseNonEmpty(env environment.Environment, s string) (string, error) { if s == "" { return "", errEmptyString } @@ -48,7 +49,7 @@ func ParseNonEmpty(s string) (string, error) { var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! // ParseValidDomain checks that s is a valid domain and returns it in lowercase -func ParseValidDomain(s string) (string, error) { +func ParseValidDomain(env environment.Environment, s string) (string, error) { if !regexpDomain.MatchString(s) { return "", errors.Errorf("%q is not a valid domain", s) } @@ -56,7 +57,7 @@ func ParseValidDomain(s string) (string, error) { } // ParseValidDomains checks that s is a comma-seperated list of valid domains and returns them in lower case -func ParseValidDomains(s string) ([]string, error) { +func ParseValidDomains(env environment.Environment, s string) ([]string, error) { if len(s) == 0 { return []string{}, nil } @@ -70,13 +71,13 @@ func ParseValidDomains(s string) ([]string, error) { } // ParseNumber parses s as a decimal integer -func ParseNumber(s string) (int, error) { +func ParseNumber(env environment.Environment, s string) (int, error) { value, err := strconv.ParseInt(s, 10, 64) return int(value), err } // ParseHttpsURL parses a string into a url that starts with 'https://' -func ParseHttpsURL(s string) (*url.URL, error) { +func ParseHttpsURL(env environment.Environment, s string) (*url.URL, error) { url, err := url.Parse(s) if err != nil { return nil, errors.Wrapf(err, "%q is not a valid URL", s) @@ -90,7 +91,7 @@ func ParseHttpsURL(s string) (*url.URL, error) { var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! // ParseEmail checks that s represents an email, and then returns it as is. -func ParseEmail(s string) (string, error) { +func ParseEmail(env environment.Environment, s string) (string, error) { if s == "" { // no email provided return "", nil } @@ -103,7 +104,7 @@ func ParseEmail(s string) (string, error) { var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! // ParseSlug parses s as a slug and returns it as is. -func ParseSlug(s string) (string, error) { +func ParseSlug(env environment.Environment, s string) (string, error) { if !regexpSlug.MatchString(s) { return "", errors.Errorf("%q is not a valid slug", s) } diff --git a/pkg/targz/targz.go b/pkg/targz/targz.go index 28dd3e3..df1db36 100644 --- a/pkg/targz/targz.go +++ b/pkg/targz/targz.go @@ -6,17 +6,18 @@ import ( "compress/gzip" "io" "io/fs" - "os" "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/pkg/environment" ) // Package packages the source directory into a 'tar.gz' file into destination. // If the destination already exists, it is truncated. // // onCopy, when not nil, is called for each file being copied into the archive. -func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) { +func Package(env environment.Environment, dst, src string, onCopy func(rel string, src string)) (count int64, err error) { // create the target archive - archive, err := os.Create(dst) + archive, err := env.Create(dst, environment.DefaultFilePerm) if err != nil { return 0, err } @@ -31,7 +32,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64, defer tarHandle.Close() // and walk through it! - err = filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error { + err = env.WalkDir(src, func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } @@ -74,7 +75,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64, } // open the file - handle, err := os.Open(path) + handle, err := env.Open(path) if err != nil { return err } diff --git a/pkg/unpack/resource.go b/pkg/unpack/resource.go index 58f7e80..d267124 100644 --- a/pkg/unpack/resource.go +++ b/pkg/unpack/resource.go @@ -3,9 +3,9 @@ package unpack import ( "io" "io/fs" - "os" "path/filepath" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/pkg/errors" ) @@ -16,9 +16,9 @@ var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a // // onInstallFile is called for each file or directory being installed. // -// If the destination path does not exist, it is created using [os.MakeDirs] +// If the destination path does not exist, it is created using [environment.MakeDirs] // The directory is installed recursively. -func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { +func InstallDir(env environment.Environment, dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { // open the source file srcFile, err := fsys.Open(src) if err != nil { @@ -43,14 +43,14 @@ func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src // do the installation of the directory. // the type cast should be safe. - return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile) + return installDir(env, dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile) } // installResource installs the resource at src within fsys to dst. // // OnInstallFile is called for each source and destination file. // OnInstallFile may be nil. -func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { +func installResource(env environment.Environment, dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { // open the srcFile srcFile, err := fsys.Open(src) if err != nil { @@ -71,19 +71,19 @@ func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, // this is a directory, so the cast is safe! if srcInfo.IsDir() { - return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile) + return installDir(env, dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile) } // this is a regular file! - return installFile(dst, srcInfo, srcFile) + return installFile(env, dst, srcInfo, srcFile) } -func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { +func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error { // create the destination - dstStat, dstErr := os.Stat(dst) + dstStat, dstErr := env.Stat(dst) switch { - case os.IsNotExist(dstErr): - if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil { + case environment.IsNotExist(dstErr): + if err := env.MkdirAll(dst, srcInfo.Mode()); err != nil { return errors.Wrapf(err, "Error creating destination directory %s", dst) } case dstErr != nil: @@ -105,6 +105,7 @@ func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src str // iterate over all the children for _, entry := range entries { if err := installResource( + env, filepath.Join(dst, entry.Name()), filepath.Join(src, entry.Name()), fsys, @@ -117,9 +118,9 @@ func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src str return nil } -func installFile(dst string, srcInfo fs.FileInfo, src fs.File) error { +func installFile(env environment.Environment, dst string, srcInfo fs.FileInfo, src fs.File) error { // create the file using the right mode! - file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode()) + file, err := env.Create(dst, srcInfo.Mode()) if err != nil { return err } diff --git a/pkg/unpack/template.go b/pkg/unpack/template.go index f4dc65a..2e7f86b 100644 --- a/pkg/unpack/template.go +++ b/pkg/unpack/template.go @@ -6,9 +6,9 @@ import ( "fmt" "io" "io/fs" - "os" "strings" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -201,7 +201,7 @@ parseloop: // Any existing file is truncated and overwritten. // // See [WriteTemplate] for possible errors. -func InstallTemplate(dst string, context map[string]string, src string, fsys fs.FS) error { +func InstallTemplate(env environment.Environment, dst string, context map[string]string, src string, fsys fs.FS) error { // open the srcFile srcFile, err := fsys.Open(src) @@ -222,7 +222,7 @@ func InstallTemplate(dst string, context map[string]string, src string, fsys fs. } // open the destination file - file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode()) + file, err := env.Create(dst, srcInfo.Mode()) if err != nil { return err } diff --git a/program.go b/program.go index 7d4d64e..cae3827 100644 --- a/program.go +++ b/program.go @@ -41,7 +41,7 @@ func NewProgram() Program { // when not running inside docker and we need a distillery // then we should warn if we are not using the distillery executable. - if dis := context.Environment; !context.Args.Flags.InternalInDocker && context.Description.Requirements.NeedsDistillery && !dis.Config.UsingDistilleryExecutable() { + if dis := context.Environment; !context.Args.Flags.InternalInDocker && context.Description.Requirements.NeedsDistillery && !dis.Config.UsingDistilleryExecutable(dis.Environment) { context.EPrintf(warnNoDeployWdcli, core.Executable, dis.Config.ExecutablePath()) }