rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'environment' package

Browse source 
This commit adds a new environment package that manages all calls to the
underlying operating system.
This commit is contained in:
Tom Wiesing 2022-09-18 14:24:22 +02:00
parent 822c70cd69
commit f19619ef9f
No known key found for this signature in database
60 changed files with 539 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

4
TODO.md
View file

@ -5,12 +5,10 @@
- Why a factory?
- First steps after provisioning
- Use `environment.Dial()` and `environment.Listen()`
- Move `provision_entrypoint.sh` into go
- Enhance Snapshots
- Export the Docker Images
- Avoid running `docker compose` executable and shift it to a library
- Move resolver code into this
- Cleanup code: Have consistent error handling
- Add a proper metadata / statistics server
- Single Malt Mode: Support having a single instance only!

14
cmd/backup.go
View file

@ -1,12 +1,10 @@
package cmd
import (
"io/fs"
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/backup"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
"github.com/tkw1536/goprogram/exit"
@ -59,7 +57,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
}
defer func() {
logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
os.RemoveAll(sPath)
dis.Environment.RemoveAll(sPath)
}()
} else {
// staging mode: use dest as a destination
@ -73,8 +71,8 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
// create the directory (if it doesn't already exist)
logging.LogMessage(context.IOStream, "Creating staging directory")
err = os.Mkdir(sPath, fs.ModePerm)
if !os.IsExist(err) && err != nil {
err = dis.Core.Environment.Mkdir(sPath, environment.DefaultDirPerm)
if !environment.IsExist(err) && err != nil {
return errSnapshotFailed.WithMessageF(err)
}
err = nil
@ -86,7 +84,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
Dest: sPath,
Auto: bk.Positionals.Dest == "",
})
backup.WriteReport(context.IOStream)
backup.WriteReport(dis.Core.Environment, context.IOStream)
return nil
}, context.IOStream, "Generating Backup")
@ -108,7 +106,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
context.IOStream.Println(archivePath)
count, err = targz.Package(archivePath, sPath, func(dst, src string) {
count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) {
context.Printf("\033[2K\r%s", dst)
})
context.Println("")

4
cmd/blind_update.go
View file

@ -3,7 +3,7 @@ package cmd
import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
)
@ -47,7 +47,7 @@ func (bu blindUpdate) Run(context wisski_distillery.Context) error {
code, err := instance.Shell(context.IOStream, "/runtime/blind_update.sh")
if err != nil {
return errBlindUpdateFailed.WithMessageF(instance.Slug, execx.ExecCommandError)
return errBlindUpdateFailed.WithMessageF(instance.Slug, environment.ExecCommandError)
}
if code != 0 {
return errBlindUpdateFailed.WithMessageF(instance.Slug, code)

32
cmd/bootstrap.go
View file

@ -2,12 +2,12 @@ package cmd
import (
"io/fs"
"os"
"path/filepath"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/exit"
@ -67,11 +67,15 @@ var errBootstrapCreateFile = exit.Error{
}
func (bs bootstrap) Run(context wisski_distillery.Context) error {
// installation environment is the native environment!
// TODO: Should this be configurable?
var env environment.Native
root := bs.Directory
// check that we didn't get a different base directory
{
got, err := core.ReadBaseDirectory()
got, err := core.ReadBaseDirectory(env)
if err == nil && got != "" && got != root {
return errBootstrapDifferent.WithMessageF(got)
}
@ -79,10 +83,10 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
{
logging.LogMessage(context.IOStream, "Creating root deployment directory")
if err := os.MkdirAll(root, fs.ModeDir); err != nil {
if err := env.MkdirAll(root, environment.DefaultDirPerm); err != nil {
return errBootstrapFailedToCreateDirectory.WithMessageF(root)
}
if err := core.WriteBaseDirectory(root); err != nil {
if err := core.WriteBaseDirectory(env, root); err != nil {
return errBootstrapFailedToSaveDirectory.WithMessageF(root)
}
context.Println(root)
@ -98,18 +102,18 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
tpl.DefaultDomain = bs.Hostname
// and use thge defaults
if err := tpl.SetDefaults(); err != nil {
if err := tpl.SetDefaults(env); err != nil {
return errBootstrapWriteConfig.WithMessageF(err)
}
{
logging.LogMessage(context.IOStream, "Copying over wdcli executable")
exe, err := os.Executable()
exe, err := env.Executable()
if err != nil {
return errBoostrapFailedToCopyExe.WithMessageF(err)
}
err = fsx.CopyFile(wdcliPath, exe)
err = fsx.CopyFile(env, wdcliPath, exe)
if err != nil && err != fsx.ErrCopySameFile {
return errBoostrapFailedToCopyExe.WithMessageF(err)
}
@ -117,9 +121,9 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
}
{
if !fsx.IsFile(envPath) {
if !fsx.IsFile(env, envPath) {
if err := logging.LogOperation(func() error {
env, err := os.Create(envPath)
env, err := env.Create(envPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -133,7 +137,8 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
context.Println(tpl.SelfOverridesFile)
if err := os.WriteFile(
if err := environment.WriteFile(
env,
tpl.SelfOverridesFile,
core.DefaultOverridesJSON,
fs.ModePerm,
@ -142,7 +147,8 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
}
context.Println(tpl.AuthorizedKeys)
if err := os.WriteFile(
if err := environment.WriteFile(
env,
tpl.AuthorizedKeys,
core.DefaultAuthorizedKeys,
fs.ModePerm,
@ -160,14 +166,14 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
// re-read the configuration and print it!
logging.LogMessage(context.IOStream, "Configuration is now complete")
f, err := os.Open(envPath)
f, err := env.Open(envPath)
if err != nil {
return errBootstrapOpenConfig.WithMessageF(err)
}
defer f.Close()
var cfg config.Config
if err := cfg.Unmarshal(f); err != nil {
if err := cfg.Unmarshal(env, f); err != nil {
return errBootstrapOpenConfig.WithMessageF(err)
}
context.Println(cfg)

11
cmd/monday.go
View file

@ -1,10 +1,10 @@
package cmd
import (
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
)
@ -29,13 +29,10 @@ func (monday) Description() wisski_distillery.Description {
}
func (monday monday) AfterParse() error {
_, err := os.Stat(monday.Positionals.GraphdbZip)
if os.IsNotExist(err) {
// TODO: Use a generic environment here!
if !fsx.IsFile(environment.Native{}, monday.Positionals.GraphdbZip) {
return errNoGraphDBZip.WithMessageF(monday.Positionals.GraphdbZip)
}
if err != nil {
return err
}
return nil
}

2
cmd/provision.go
View file

@ -57,7 +57,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
// check that the base directory does not exist
logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
if fsx.IsDirectory(instance.FilesystemBase) {
if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) {
return errProvisionAlreadyExists.WithMessageF(slug)
}

4
cmd/purge.go
View file

@ -1,8 +1,6 @@
package cmd
import (
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
@ -73,7 +71,7 @@ func (p purge) Run(context wisski_distillery.Context) error {
// remove the filesystem
logging.LogMessage(context.IOStream, "Removing from filesystem %s", instance.FilesystemBase)
if err := os.RemoveAll(instance.FilesystemBase); err != nil {
if err := dis.Core.Environment.RemoveAll(instance.FilesystemBase); err != nil {
context.EPrintln(err)
}

6
cmd/rebuild.go
View file

@ -33,7 +33,9 @@ var errRebuildFailed = exit.Error{
}
func (rb rebuild) Run(context wisski_distillery.Context) error {
instances, err := context.Environment.Instances().Load(rb.Positionals.Slug...)
dis := context.Environment
instances, err := dis.Instances().Load(rb.Positionals.Slug...)
if err != nil {
return err
}
@ -44,7 +46,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error {
logging.LogOperation(func() error {
s := instance.Barrel()
if err := logging.LogOperation(func() error {
return s.Install(context.IOStream, component.InstallationContext{})
return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
}, context.IOStream, "Installing docker stack"); err != nil {
globalErr = err
return err

4
cmd/reserve.go
View file

@ -58,7 +58,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
// check that the base directory does not exist
logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
if fsx.IsDirectory(instance.FilesystemBase) {
if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) {
return errProvisionAlreadyExists.WithMessageF(slug)
}
@ -66,7 +66,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
s := instance.Reserve()
{
if err := logging.LogOperation(func() error {
return s.Install(context.IOStream, component.InstallationContext{})
return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
}, context.IOStream, "Installing docker stack"); err != nil {
return err
}

12
cmd/snapshot.go
View file

@ -2,11 +2,11 @@ package cmd
import (
"io/fs"
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
"github.com/tkw1536/goprogram/exit"
@ -60,7 +60,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
}
defer func() {
logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
os.RemoveAll(sPath)
dis.Core.Environment.RemoveAll(sPath)
}()
} else {
// staging mode: use dest as a destination
@ -74,8 +74,8 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// create the directory (if it doesn't already exist)
logging.LogMessage(context.IOStream, "Creating staging directory")
err = os.Mkdir(sPath, fs.ModePerm)
if !os.IsExist(err) && err != nil {
err = dis.Core.Environment.Mkdir(sPath, fs.ModePerm)
if !environment.IsExist(err) && err != nil {
return errSnapshotFailed.WithMessageF(err)
}
err = nil
@ -92,7 +92,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
})
// write out the report, ignoring any errors!
sreport.WriteReport(context.IOStream)
sreport.WriteReport(dis.Core.Environment, context.IOStream)
return nil
}, context.IOStream, "Generating Snapshot")
@ -115,7 +115,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
context.IOStream.Println(archivePath)
count, err = targz.Package(archivePath, sPath, func(dst, src string) {
count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) {
context.Printf("\033[2K\r%s", dst)
})
context.Println("")

23
cmd/system_update.go
View file

@ -1,13 +1,12 @@
package cmd
import (
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/tkw1536/goprogram/exit"
@ -43,13 +42,10 @@ var errNoGraphDBZip = exit.Error{
}
func (s systemupdate) AfterParse() error {
_, err := os.Stat(s.Positionals.GraphdbZip)
if os.IsNotExist(err) {
// TODO: Use a generic environment here!
if !fsx.IsFile(environment.Native{}, s.Positionals.GraphdbZip) {
return errNoGraphDBZip.WithMessageF(s.Positionals.GraphdbZip)
}
if err != nil {
return err
}
return nil
}
@ -85,7 +81,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
dis.SnapshotsArchivePath(),
} {
context.Println(d)
if err := os.MkdirAll(d, os.ModeDir); err != nil {
if err := dis.Core.Environment.MkdirAll(d, environment.DefaultDirPerm); err != nil {
return errBoostrapFailedToCreateDirectory.WithMessageF(d, err)
}
}
@ -123,10 +119,10 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
for _, component := range dis.Installables() {
stack := component.Stack()
stack := component.Stack(dis.Core.Environment)
ctx := component.Context(ctx)
if err := logging.LogOperation(func() error {
return stack.Install(context.IOStream, ctx)
return stack.Install(dis.Core.Environment, context.IOStream, ctx)
}, context.IOStream, "Installing docker stack %q", component.Name()); err != nil {
return err
}
@ -143,7 +139,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
}
if err := logging.LogOperation(func() error {
return unpack.InstallDir(dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) {
return unpack.InstallDir(dis.Core.Environment, dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) {
context.Printf("[copy] %s\n", dst)
})
}, context.IOStream, "Unpacking Runtime Components"); err != nil {
@ -175,10 +171,11 @@ var errMustExecFailed = exit.Error{
// mustExec indicates that the given executable process must complete successfully.
// If it does not, returns errMustExecFailed
func (si systemupdate) mustExec(context wisski_distillery.Context, workdir string, exe string, argv ...string) error {
dis := context.Environment
if workdir == "" {
workdir = context.Environment.Config.DeployRoot
}
code := execx.Exec(context.IOStream, workdir, exe, argv...)
code := dis.Core.Environment.Exec(context.IOStream, workdir, exe, argv...)
if code != 0 {
err := errMustExecFailed.WithMessageF(code)
err.ExitCode = exit.ExitCode(code)

10
cmd/update_prefix_config.go
View file

@ -1,11 +1,11 @@
package cmd
import (
"io/fs"
"os"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/exit"
)
@ -42,7 +42,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
target := ddis.ResolverConfigPath()
// print the configuration
config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm)
config, err := dis.Core.Environment.Create(target, environment.DefaultFilePerm)
if err != nil {
return errPrefixUpdateFailed.WithMessageF(err)
}
@ -58,7 +58,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
context.IOStream.Printf("%s", data)
// and write it out!
if _, err := config.WriteString(data); err != nil {
if _, err := io.WriteString(config, data); err != nil {
return err
}
@ -70,7 +70,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
// and restart the resolver to apply the config!
logging.LogMessage(context.IOStream, "restarting resolver stack")
if err := ddis.Stack().Restart(context.IOStream); err != nil {
if err := ddis.Stack(ddis.Environment).Restart(context.IOStream); err != nil {
return errPrefixUpdateFailed.WithMessageF(err)
}

8
internal/backup/backup.go
View file

@ -2,14 +2,13 @@
package backup
import (
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
@ -61,6 +60,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
err: bc.Backup(context),
}
}(bc, &context{
env: dis.Core.Environment,
io: io,
dst: filepath.Join(backup.Description.Dest, bc.BackupName()),
files: files,
@ -73,7 +73,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
defer wg.Done()
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(instancesBackupDir, environment.DefaultDirPerm); err != nil {
backup.InstanceListErr = err
return
}
@ -89,7 +89,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
for i, instance := range instances {
backup.InstanceSnapshots[i] = func() wisski.Snapshot {
dir := filepath.Join(instancesBackupDir, instance.Slug)
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil {
return wisski.Snapshot{
ErrPanic: err,
}

10
internal/backup/context.go
View file

@ -3,16 +3,16 @@ package backup
import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/stream"
)
// context implements [components.BackupContext]
type context struct {
env environment.Environment
io stream.IOStream
dst string // destination directory
files chan string // files channel
@ -54,7 +54,7 @@ func (bc *context) AddDirectory(path string, op func() error) error {
}
// run the make directory
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
if err := bc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil {
return err
}
@ -72,7 +72,7 @@ func (bc *context) CopyFile(dst, src string) error {
return err
}
bc.sendPath(dst)
return fsx.CopyFile(dstPath, src)
return fsx.CopyFile(bc.env, dstPath, src)
}
func (bc *context) AddFile(path string, op func(file io.Writer) error) error {
@ -83,7 +83,7 @@ func (bc *context) AddFile(path string, op func(file io.Writer) error) error {
}
// create the file
file, err := os.Create(dst)
file, err := bc.env.Create(dst, environment.DefaultFilePerm)
if err != nil {
return err
}

12
internal/backup/report.go
View file

@ -4,13 +4,13 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/stream"
)
@ -96,20 +96,20 @@ func (backup Backup) Report(w io.Writer) (int, error) {
// WriteReport writes out the report belonging to this backup.
// It is a separate function, to allow writing it indepenently of the rest.
func (backup Backup) WriteReport(io stream.IOStream) error {
func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
io.Println(reportPath)
stream.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = report.WriteString(backup.String())
_, err = io.WriteString(report, backup.String())
return err
}, io, "Writing backup report")
}, stream, "Writing backup report")
}

4
internal/component/backup.go
View file

@ -28,8 +28,8 @@ type BackupContext interface {
// It then allows op to fill the file.
AddDirectory(path string, op func() error) error
// CopyFile copies a file from source to dst.
CopyFile(dest, src string) error
// CopyFile copies a file from src to dst.
CopyFile(dst, src string) error
// AddFile creates a new file at the provided path inside the destination.
// Passing the empty path creates the destination as a file.

16
internal/component/component.go
View file

@ -3,6 +3,7 @@ package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Component represents a logical subsystem of the distillery.
@ -38,7 +39,7 @@ type Installable interface {
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call [ComponentBase.MakeStack]
Stack() StackWithResources
Stack(env environment.Environment) StackWithResources
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
@ -47,8 +48,14 @@ type Installable interface {
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
Config *config.Config // Config is the configuration of the underlying distillery
Core // the core of the associated distillery
Dir string // Dir is the directory this component lives in
}
// Core represents the core of a distillery
type Core struct {
Environment environment.Environment // environment to use for reading / writing to and from the distillery
Config *config.Config // the configuration of the distillery
}
// Base returns a reference to the ComponentBase
@ -67,7 +74,8 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext {
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources {
func (cb ComponentBase) MakeStack(env environment.Environment, stack StackWithResources) StackWithResources {
stack.Env = env
stack.Dir = cb.Dir
return stack
}

7
internal/component/control/control.go
View file

@ -6,6 +6,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Control represents the control server
@ -24,8 +25,8 @@ func (control Control) Name() string {
//go:embed all:control control.env
var resources embed.FS
func (control Control) Stack() component.StackWithResources {
return control.ComponentBase.MakeStack(component.StackWithResources{
func (control Control) Stack(env environment.Environment) component.StackWithResources {
return control.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
@ -49,6 +50,6 @@ func (control Control) Stack() component.StackWithResources {
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: control.Config.CurrentExecutable(),
core.Executable: control.Config.CurrentExecutable(control.Environment), // TODO: Does this make sense?
}
}

3
internal/component/control/resolver.go
View file

@ -2,7 +2,6 @@ package control
import (
"fmt"
"os"
"path/filepath"
"regexp"
@ -37,7 +36,7 @@ func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler,
// open the prefix file
prefixFile := control.ResolverConfigPath()
fs, err := os.Open(prefixFile)
fs, err := control.Environment.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err

3
internal/component/control/self.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"github.com/tkw1536/goprogram/stream"
@ -13,7 +12,7 @@ import (
// self returns the handler for the self overrides
func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(control.Config.SelfOverridesFile)
overrides, err := control.Environment.Open(control.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
if err != nil {
return redirect, err

16
internal/component/installable.go
View file

@ -2,9 +2,9 @@ package component
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/pkg/errors"
@ -29,7 +29,7 @@ type StackWithResources struct {
CopyContextFiles []string // Files to copy from the installation context
MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
MakeDirs []string // directories to ensure that exist
TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
@ -42,10 +42,11 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(
env,
is.Dir,
is.ContextPath,
is.Resources,
@ -62,6 +63,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
if err := unpack.InstallTemplate(
env,
envDest,
is.EnvContext,
is.EnvPath,
@ -78,9 +80,9 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
io.Printf("[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = fs.ModeDir
is.MakeDirsPerm = environment.DefaultDirPerm
}
if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
return err
}
}
@ -98,7 +100,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
// copy over file from context
io.Printf("[copy] %s (from %s)\n", dst, src)
if err := fsx.CopyFile(dst, src); err != nil {
if err := fsx.CopyFile(env, dst, src); err != nil {
return errors.Wrapf(err, "Unable to copy file %s", src)
}
}
@ -109,7 +111,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
dst := filepath.Join(is.Dir, name)
io.Printf("[touch] %s\n", dst)
if err := fsx.Touch(dst); err != nil {
if err := fsx.Touch(env, dst); err != nil {
return err
}
}

4
internal/component/instances/wisski_create.go
View file

@ -21,7 +21,7 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
wisski.instances = instances
// make sure that the slug is valid!
slug, err = stringparser.ParseSlug(slug)
slug, err = stringparser.ParseSlug(instances.Environment, slug)
if err != nil {
return wisski, errInvalidSlug
}
@ -70,7 +70,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Barrel()
if err := st.Install(io, component.InstallationContext{}); err != nil {
if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
return err
}

5
internal/component/instances/wisski_pathbuilders.go
View file

@ -2,12 +2,11 @@ package instances
import (
"fmt"
"io/fs"
"os"
"path/filepath"
_ "embed"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@ -51,7 +50,7 @@ func (wisski *WissKI) ExportPathbuilders(dest string) error {
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
if err := environment.WriteFile(wisski.instances.Core.Environment, name, pbxml, environment.DefaultFilePerm); err != nil {
return err
}
}

7
internal/component/instances/wisski_prefix.go
View file

@ -3,7 +3,6 @@ package instances
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
@ -14,7 +13,7 @@ import (
// NoPrefix checks if this WissKI instance is excluded from generating prefixes.
// TODO: Move this to the database!
func (wisski *WissKI) NoPrefix() bool {
return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
@ -41,8 +40,8 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) {
// custom prefixes
prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if fsx.IsFile(wisski.instances.Environment, prefixPath) {
prefix, err := wisski.instances.Core.Environment.Open(prefixPath)
if err != nil {
return "", err
}

6
internal/component/instances/wisski_stack.go
View file

@ -2,7 +2,6 @@ package instances
import (
"embed"
"io/fs"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component"
@ -16,6 +15,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: barrelResources,
@ -35,8 +35,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
"GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
@ -52,6 +51,7 @@ func (wisski WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: reserveResources,

2
internal/component/sql/backup.go
View file

@ -17,7 +17,7 @@ func (*SQL) BackupName() string {
func (sql *SQL) Backup(context component.BackupContext) error {
return context.AddFile("", func(file io.Writer) error {
io := context.IO().Streams(file, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}

5
internal/component/sql/database.go
View file

@ -21,6 +21,7 @@ func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, err
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
DefaultStringSize: 256,
}
// TODO: Use sql.Core.Environment.Dial
db, err := gorm.Open(mysql.New(cfg), config)
if err != nil {
@ -63,7 +64,7 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error {
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}
@ -75,7 +76,7 @@ func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) err
// OpenShell executes a mysql shell command
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)
return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
}
// WaitShell waits for the sql database to be reachable via a docker-compose shell

8
internal/component/sql/sql.go
View file

@ -3,10 +3,10 @@ package sql
import (
"context"
"embed"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SQL struct {
@ -25,12 +25,12 @@ func (SQL) Name() string {
//go:embed all:sql
var resources embed.FS
func (ssh SQL) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SQL) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "sql",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirsPerm: environment.DefaultDirPerm,
MakeDirs: []string{
"data",
},

5
internal/component/ssh/ssh.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SSH struct {
@ -17,8 +18,8 @@ func (SSH) Name() string {
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SSH) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
})

11
internal/component/stack.go
View file

@ -6,7 +6,7 @@ import (
"bytes"
"errors"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
)
@ -18,6 +18,7 @@ import (
type Stack struct {
Dir string // Directory this Stack is located in
Env environment.Environment
DockerExecutable string // Path to the native docker executable to use
}
@ -101,7 +102,7 @@ func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string
code, err := ds.compose(io, compose...)
if err != nil {
return execx.ExecCommandError, nil
return environment.ExecCommandError, nil
}
return code, nil
}
@ -175,10 +176,10 @@ func (ds Stack) Down(io stream.IOStream) error {
func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
if ds.DockerExecutable == "" {
var err error
ds.DockerExecutable, err = execx.LookPathAbs("docker")
ds.DockerExecutable, err = ds.Env.LookPathAbs("docker")
if err != nil {
return execx.ExecCommandError, err
return environment.ExecCommandError, err
}
}
return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
}

11
internal/component/triplestore/database.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
@ -61,6 +62,14 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
}
// create the request object
client := &http.Client{
Transport: &http.Transport{
Dial: ts.Environment.Dial,
DialTLS: func(network, addr string) (net.Conn, error) {
return nil, errors.New("not implemented")
},
},
}
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
if err != nil {
return nil, err
@ -76,7 +85,7 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
// and send it
return http.DefaultClient.Do(req)
return client.Do(req)
}
// Wait waits for the connection to the Triplestore to succeed.

7
internal/component/triplestore/triplestore.go
View file

@ -3,11 +3,11 @@ package triplestore
import (
"context"
"embed"
"io/fs"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type Triplestore struct {
@ -26,14 +26,13 @@ func (Triplestore) Name() string {
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() component.StackWithResources {
return ts.ComponentBase.MakeStack(component.StackWithResources{
func (ts Triplestore) Stack(env environment.Environment) component.StackWithResources {
return ts.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),

15
internal/component/web/web.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Web implements the ingress gateway for the distillery.
@ -17,11 +18,11 @@ func (Web) Name() string {
return "web"
}
func (web Web) Stack() component.StackWithResources {
func (web Web) Stack(env environment.Environment) component.StackWithResources {
if web.Config.HTTPSEnabled() {
return web.stackHTTPS()
return web.stackHTTPS(env)
} else {
return web.stackHTTP()
return web.stackHTTP(env)
}
}
@ -29,8 +30,8 @@ func (web Web) Stack() component.StackWithResources {
//go:embed web-https.env
var httpsResources embed.FS
func (web Web) stackHTTPS() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTPS(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpsResources,
ContextPath: "web-https",
EnvPath: "web-https.env",
@ -45,8 +46,8 @@ func (web Web) stackHTTPS() component.StackWithResources {
//go:embed web-http.env
var httpResources embed.FS
func (web Web) stackHTTP() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTP(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpResources,
ContextPath: "web-http",
EnvPath: "web-http.env",

14
internal/config/executable.go
View file

@ -1,10 +1,10 @@
package config
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
@ -14,19 +14,19 @@ func (cfg Config) ExecutablePath() string {
}
// UsingDistilleryExecutable checks if the current process is using the distillery executable
func (cfg Config) UsingDistilleryExecutable() bool {
exe, err := os.Executable()
func (cfg Config) UsingDistilleryExecutable(env environment.Environment) bool {
exe, err := env.Executable()
if err != nil {
return false
}
return fsx.SameFile(exe, cfg.ExecutablePath())
return fsx.SameFile(env, exe, cfg.ExecutablePath())
}
// CurrentExecutable returns the path to the current executable being used.
// When it does not exist, falls back to the default executable.
func (cfg Config) CurrentExecutable() string {
exe, err := os.Executable()
if err != nil || !fsx.IsFile(exe) {
func (cfg Config) CurrentExecutable(env environment.Environment) string {
exe, err := env.Executable()
if err != nil || !fsx.IsFile(env, exe) {
return cfg.ExecutablePath()
}
return exe

19
internal/config/read.go
View file

@ -4,6 +4,7 @@ import (
"io"
"reflect"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/envreader"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
)
@ -16,7 +17,7 @@ import (
// When a key is missing, it is set to the default value.
//
// See also [stringparser.Parse].
func (config *Config) Unmarshal(src io.Reader) error {
func (config *Config) Unmarshal(env environment.Environment, src io.Reader) error {
// read all the values!
values, err := envreader.ReadAll(src)
if err != nil {
@ -32,26 +33,26 @@ func (config *Config) Unmarshal(src io.Reader) error {
tField := tConfig.Field(i)
vField := vConfig.FieldByName(tField.Name)
env := tField.Tag.Get("env")
dflt := tField.Tag.Get("default")
parser := tField.Tag.Get("parser")
tEnv := tField.Tag.Get("env")
tDefault := tField.Tag.Get("default")
tParser := tField.Tag.Get("parser")
// skip it if it isn't loaded!
if env == "" {
if tEnv == "" {
continue
}
// read the value with a default
value, ok := values[env]
value, ok := values[tEnv]
if !ok || value == "" {
if dflt == "" {
if tDefault == "" {
continue
}
value = dflt
value = tDefault
}
// parse the value!
if err := stringparser.Parse(parser, value, vField); err != nil {
if err := stringparser.Parse(env, tParser, value, vField); err != nil {
return err
}
}

5
internal/config/template.go
View file

@ -7,6 +7,7 @@ import (
"reflect"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/hostname"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
@ -29,13 +30,13 @@ type Template struct {
}
// SetDefaults sets defaults on the template
func (tpl *Template) SetDefaults() (err error) {
func (tpl *Template) SetDefaults(env environment.Environment) (err error) {
if tpl.DeployRoot == "" {
tpl.DeployRoot = core.BaseDirectoryDefault
}
if tpl.DefaultDomain == "" {
tpl.DefaultDomain = hostname.FQDN()
tpl.DefaultDomain = hostname.FQDN(env)
}
if tpl.SelfOverridesFile == "" {

11
internal/core/meta.go
View file

@ -3,10 +3,11 @@ package core
import (
"errors"
"io/fs"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// MetaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable.
@ -33,7 +34,7 @@ var errReadBaseDirectoryEmpty = errors.New("ReadBaseDirectory: Directory is empt
// Use [ParamsFromEnv] to initialize parameters completely.
//
// It does not perform any reading of files.
func ReadBaseDirectory() (value string, err error) {
func ReadBaseDirectory(env environment.Environment) (value string, err error) {
// get the path!
path, err := MetaConfigPath()
if err != nil {
@ -41,7 +42,7 @@ func ReadBaseDirectory() (value string, err error) {
}
// read the meta config file!
contents, err := os.ReadFile(path)
contents, err := environment.ReadFile(env, path)
if err != nil {
return "", err
}
@ -59,7 +60,7 @@ func ReadBaseDirectory() (value string, err error) {
}
// WriteBaseDirectory writes the base directory to the environment, or returns an error
func WriteBaseDirectory(dir string) error {
func WriteBaseDirectory(env environment.Environment, dir string) error {
// get the path!
path, err := MetaConfigPath()
if err != nil {
@ -67,5 +68,5 @@ func WriteBaseDirectory(dir string) error {
}
// just put the directory inside it!
return os.WriteFile(path, []byte(dir), fs.ModePerm)
return environment.WriteFile(env, path, []byte(dir), fs.ModePerm)
}

7
internal/core/params.go
View file

@ -1,8 +1,9 @@
package core
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Params are used to initialize the excutable.
@ -15,9 +16,9 @@ type Params struct {
func ParamsFromEnv() (params Params, err error) {
// try to read the base directory!
value, err := ReadBaseDirectory()
value, err := ReadBaseDirectory(environment.Native{}) // TODO: Are we sure about the native environment here?
switch {
case os.IsNotExist(err):
case environment.IsNotExist(err):
params.ConfigPath = BaseDirectoryDefault
case err == nil:
params.ConfigPath = value

5
internal/wisski/backup_prune.go
View file

@ -1,7 +1,6 @@
package wisski
import (
"os"
"path/filepath"
"time"
@ -18,7 +17,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.SnapshotsArchivePath()
// list all the files
entries, err := os.ReadDir(sPath)
entries, err := dis.Core.Environment.ReadDir(sPath)
if err != nil {
return err
}
@ -44,7 +43,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
if err := os.Remove(path); err != nil {
if err := dis.Core.Environment.Remove(path); err != nil {
return err
}
}

2
internal/wisski/component.go
View file

@ -58,7 +58,7 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
}
base := c.Base()
base.Config = dis.Config
base.Core = dis.Core
if base.Dir == "" {
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", c.Name())
}

9
internal/wisski/distillery.go
View file

@ -3,16 +3,15 @@ package wisski
import (
"context"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
// Distillery represents a WissKI Distillery
//
// It is the main structure used to interact with different components.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// core holds the core of the distillery
component.Core
// Upstream holds information to connect to the various running
// distillery components.
@ -25,7 +24,7 @@ type Distillery struct {
components
}
// Upstream are the upstream urls connecting to the various external components.
// Upstream contains the configuration for accessing remote configuration.
type Upstream struct {
SQL string
Triplestore string

13
internal/wisski/init.go
View file

@ -1,10 +1,10 @@
package wisski
import (
"os"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
)
@ -21,6 +21,9 @@ var errOpenConfig = exit.Error{
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
Core: component.Core{
Environment: environment.Native{},
},
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
@ -34,7 +37,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
if flags.InternalInDocker {
dis.Upstream.SQL = "sql:3306"
dis.Upstream.Triplestore = "triplestore:7200"
params.ConfigPath = os.Getenv("CONFIG_PATH")
params.ConfigPath = dis.Core.Environment.GetEnv("CONFIG_PATH")
}
// if we don't need to load the config, there is nothing to do
@ -52,7 +55,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
}
// open the config file!
f, err := os.Open(params.ConfigPath)
f, err := dis.Core.Environment.Open(params.ConfigPath)
if err != nil {
return nil, errOpenConfig.WithMessageF(err)
}
@ -62,6 +65,6 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
dis.Config = &config.Config{
ConfigPath: cfg,
}
err = dis.Config.Unmarshal(f)
err = dis.Config.Unmarshal(dis.Core.Environment, f)
return
}

29
internal/wisski/snapshot.go
View file

@ -4,8 +4,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
@ -13,6 +11,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
@ -42,7 +41,7 @@ func (dis *Distillery) SnapshotsArchivePath() string {
// The path is guaranteed to not exist.
func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
// TODO: Consider moving these into a subdirectory with the provided prefix.
for path == "" || fsx.Exists(path) {
for path == "" || fsx.Exists(dis.Environment, path) {
name := dis.newSnapshotName(prefix) + ".tar.gz"
path = filepath.Join(dis.SnapshotsArchivePath(), name)
}
@ -64,9 +63,9 @@ func (*Distillery) newSnapshotName(prefix string) string {
// NewSnapshotStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
for path == "" || os.IsExist(err) {
for path == "" || environment.IsExist(err) {
path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
err = os.Mkdir(path, os.ModeDir)
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
}
if err != nil {
path = ""
@ -210,7 +209,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
files <- bkPath
info, err := os.Create(bkPath)
info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -227,7 +226,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
// copy over whatever is in the base directory
return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
files <- dst
})
}, &snapshot.ErrFilesystem)
@ -237,7 +236,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
files <- tsPath
nquads, err := os.Create(tsPath)
nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -253,7 +252,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
files <- sqlPath
sql, err := os.Create(sqlPath)
sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -279,7 +278,7 @@ func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, inst
files <- pbPath
// create the directory!
if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
return err
}
@ -312,20 +311,20 @@ func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[stri
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
io.Println(reportPath)
stream.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = report.WriteString(snapshot.String())
_, err = io.WriteString(report, snapshot.String())
return err
}, io, "Writing snapshot report")
}, stream, "Writing snapshot report")
}

58
pkg/environment/environment.go Normal file
View file

@ -0,0 +1,58 @@
package environment
import (
"io"
"io/fs"
"net"
"time"
"github.com/tkw1536/goprogram/stream"
)
// Environment represents an environment that a program can run it.
// It mostly mimics the interfaces of the [os] package.
type Environment interface {
isEnv()
GetEnv(name string) string
Stat(path string) (fs.FileInfo, error)
Lstat(path string) (fs.FileInfo, error)
Readlink(path string) (string, error)
Symlink(oldname, newname string) error
ReadDir(name string) ([]fs.DirEntry, error)
Open(path string) (fs.File, error)
Chtimes(name string, atime time.Time, mtime time.Time) error
SameFile(f1, f2 fs.FileInfo) bool
Create(path string, mode fs.FileMode) (WritableFile, error)
Mkdir(path string, mode fs.FileMode) error
MkdirAll(path string, mode fs.FileMode) error
Remove(path string) error
RemoveAll(path string) error
WalkDir(root string, fn fs.WalkDirFunc) error
Abs(path string) (string, error)
Listen(network, address string) (net.Listener, error)
Dial(network, address string) (net.Conn, error)
Executable() (string, error)
Exec(io stream.IOStream, workdir string, exe string, argv ...string) int
LookPathAbs(name string) (string, error)
}
type WritableFile interface {
fs.File
io.Writer
}
func init() {
var _ Environment = Native{}
}

69
pkg/environment/globals.go Normal file
View file

@ -0,0 +1,69 @@
package environment
import (
"bytes"
"io"
"io/fs"
"os"
"github.com/tkw1536/goprogram/stream"
)
// ExecCommandError is returned by Exec when a command could not be executed.
// This typically hints that the executable cannot be found, but may have other causes.
const ExecCommandError = 127
// DefaultFilePerm is the default mode to use for files
const DefaultFilePerm fs.FileMode = 0666
// DefaultDirPerm is the default mode to use for directories
const DefaultDirPerm fs.FileMode = os.ModeDir & fs.ModePerm
// IsExist checks if the provided error represents a 'does not exist' errror
func IsExist(err error) bool {
return os.IsExist(err)
}
// IsNotExist checks if the provided error represents a 'does exist' error
func IsNotExist(err error) bool {
return os.IsNotExist(err)
}
// WriteFile is like [os.WriteFile].
func WriteFile(env Environment, path string, data []byte, mode fs.FileMode) error {
handle, err := env.Create(path, mode)
if err != nil {
return err
}
defer handle.Close()
if _, err := handle.Write(data); err != nil {
return err
}
return nil
}
// ReadFile is like [os.ReadFile]
func ReadFile(env Environment, path string) ([]byte, error) {
// open the file!
file, err := env.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// copy everything into a buffer!
var buffer bytes.Buffer
if _, err := io.Copy(&buffer, file); err != nil {
return nil, err
}
// return the buffer contents!
return buffer.Bytes(), nil
}
// MustExec is like Exec, except that it returns true if the command exited successfully, and else false.
func MustExec(env Environment, io stream.IOStream, workdir string, exe string, argv ...string) bool {
return env.Exec(io, workdir, exe, argv...) == 0
}

18
pkg/execx/exec.go → pkg/environment/native_exec.go
View file

@ -1,5 +1,4 @@
// Package execx defines extensions to the "os/exec" package
package execx
package environment
import (
"os/exec"
@ -7,15 +6,11 @@ import (
"github.com/tkw1536/goprogram/stream"
)
// ExecCommandError is returned by Exec when a command could not be executed.
// This typically hints that the executable cannot be found, but may have other causes.
const ExecCommandError = 127
// Exec executes a system command with the specified input/output streams, working directory, and arguments.
//
// If the command executes, it's exit code will be returned.
// If the command can not be executed, returns [ExecCommandError].
func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
func (Native) Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
// setup the command
cmd := exec.Command(exe, argv...)
cmd.Dir = workdir
@ -40,7 +35,10 @@ func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
return 0
}
// MustExec is like Exec, except that it returns true if the command exited successfully, and else false.
func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool {
return Exec(io, workdir, exe, argv...) == 0
func (n Native) LookPathAbs(file string) (string, error) {
path, err := exec.LookPath(file)
if err != nil {
return "", err
}
return n.Abs(path)
}

80
pkg/environment/native_fs.go Normal file
View file

@ -0,0 +1,80 @@
package environment
import (
"io/fs"
"os"
"path/filepath"
"time"
)
type Native struct{}
func (Native) isEnv() {}
func (Native) GetEnv(name string) string {
return os.Getenv(name)
}
func (Native) Stat(path string) (fs.FileInfo, error) {
return os.Stat(path)
}
func (Native) Lstat(path string) (fs.FileInfo, error) {
return os.Lstat(path)
}
func (Native) Readlink(path string) (string, error) {
return os.Readlink(path)
}
func (Native) Symlink(oldname, newname string) error {
return os.Symlink(oldname, newname)
}
func (Native) ReadDir(name string) ([]fs.DirEntry, error) {
return os.ReadDir(name)
}
func (Native) SameFile(f1, f2 fs.FileInfo) bool {
return os.SameFile(f1, f2)
}
func (Native) WalkDir(path string, f fs.WalkDirFunc) error {
return filepath.WalkDir(path, f)
}
func (Native) Executable() (string, error) {
return os.Executable() // TODO: not sure this works with the remote concepts
}
func (Native) Open(path string) (fs.File, error) {
return os.Open(path)
}
func (Native) Create(path string, mode fs.FileMode) (WritableFile, error) {
return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
}
func (Native) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
func (Native) Mkdir(path string, mode fs.FileMode) error {
return os.Mkdir(path, mode)
}
func (Native) MkdirAll(path string, mode fs.FileMode) error {
return os.MkdirAll(path, mode)
}
func (Native) Remove(path string) error {
return os.Remove(path)
}
func (Native) RemoveAll(path string) error {
return os.RemoveAll(path)
}
func (Native) Abs(path string) (string, error) {
return filepath.Abs(path)
}

11
pkg/environment/native_net.go Normal file
View file

@ -0,0 +1,11 @@
package environment
import "net"
func (Native) Listen(network, address string) (net.Listener, error) {
return net.Listen(network, address)
}
func (Native) Dial(network, address string) (net.Conn, error) {
return net.Dial(network, address)
}

15
pkg/execx/look.go
View file

@ -1,15 +0,0 @@
package execx
import (
"os/exec"
"path/filepath"
)
// LookPathAbs is like [exec.LookPath], but always returns an absolute path
func LookPathAbs(file string) (string, error) {
path, err := exec.LookPath(file)
if err != nil {
return "", err
}
return filepath.Abs(path)
}

41
pkg/fsx/copy.go
View file

@ -4,8 +4,9 @@ import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
var ErrCopySameFile = errors.New("src and dst must be different")
@ -14,13 +15,13 @@ var ErrCopySameFile = errors.New("src and dst must be different")
// When src points to a symbolic link, will copy the symbolic link.
//
// When dst and src are the same file, returns ErrCopySameFile.
func CopyFile(dst, src string) error {
if SameFile(src, dst) {
func CopyFile(env environment.Environment, dst, src string) error {
if SameFile(env, src, dst) {
return ErrCopySameFile
}
// open the source
srcFile, err := os.Open(src)
srcFile, err := env.Open(src)
if err != nil {
return err
}
@ -33,7 +34,7 @@ func CopyFile(dst, src string) error {
}
// open or create the destination
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode())
dstFile, err := env.Create(dst, srcStat.Mode())
if err != nil {
return err
}
@ -46,27 +47,27 @@ func CopyFile(dst, src string) error {
// CopyLink copies a link from src to dst.
// If dst already exists, it is deleted and then re-created.
func CopyLink(dst, src string) error {
func CopyLink(env environment.Environment, dst, src string) error {
// if they're the same file that is an error
if SameFile(dst, src) {
if SameFile(env, dst, src) {
return ErrCopySameFile
}
// read the link target
target, err := os.Readlink(src)
target, err := env.Readlink(src)
if err != nil {
return err
}
// delete it if it already exists
if Exists(dst) {
if err := os.Remove(dst); err != nil {
if Exists(env, dst) {
if err := env.Remove(dst); err != nil {
return err
}
}
// make the symbolic link!
return os.Symlink(target, dst)
return env.Symlink(target, dst)
}
var ErrDstFile = errors.New("dst is a file")
@ -77,16 +78,16 @@ var ErrDstFile = errors.New("dst is a file")
// When a directory already exists, additional files are not deleted.
//
// onCopy, when not nil, is called for each file or directory being copied.
func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
func CopyDirectory(env environment.Environment, dst, src string, onCopy func(dst, src string)) error {
// sanity checks
if SameFile(src, dst) {
if SameFile(env, src, dst) {
return ErrCopySameFile
}
if IsFile(dst) {
if IsFile(env, dst) {
return ErrDstFile
}
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
return env.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
@ -111,19 +112,19 @@ func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
}
// if we have a symbolic link, copy the link!
if info.Mode()&os.ModeSymlink != 0 {
return CopyLink(dst, path)
if info.Mode()&fs.ModeSymlink != 0 {
return CopyLink(env, dst, path)
}
// if we got a file, we should copy it normally
if !d.IsDir() {
return CopyFile(dst, path)
return CopyFile(env, dst, path)
}
// create the directory, but ignore an error if the directory already exists.
// this is so that we can copy one tree into another tree.
err = os.Mkdir(dst, info.Mode())
if os.IsExist(err) && IsDirectory(dst) {
err = env.Mkdir(dst, info.Mode())
if environment.IsExist(err) && IsDirectory(env, dst) {
err = nil
}

29
pkg/fsx/same.go
View file

@ -1,17 +1,18 @@
package fsx
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// SameFile checks if path1 and path2 refer to the same file.
// If both files exist, they are compared using [os.SameFile].
// If both files exist, they are compared using [env.SameFile].
// If both files do not exist, the paths are first compared syntactically and then via recursion on [filepath.Dir].
func SameFile(path1, path2 string) bool {
func SameFile(env environment.Environment, path1, path2 string) bool {
// initial attempt: check if directly
same, certain := couldBeSameFile(path1, path2)
same, certain := couldBeSameFile(env, path1, path2)
if certain {
return same
}
@ -28,28 +29,28 @@ func SameFile(path1, path2 string) bool {
// compare the base names!
{
same, _ := couldBeSameFile(d1, d2)
same, _ := couldBeSameFile(env, d1, d2)
return same
}
}
// couldBeSameFile checks if path1 might be the same as path2.
//
// If both files exist, compares using [os.SameFile].
// If both files exist, compares using [env.SameFile].
// Otherwise compares absolute paths using string comparison.
//
// same indicates if they might be the same file.
// authorative indiciates if the result is authorative.
func couldBeSameFile(path1, path2 string) (same, authorative bool) {
func couldBeSameFile(env environment.Environment, path1, path2 string) (same, authorative bool) {
{
// stat both files
info1, err1 := os.Stat(path1)
info2, err2 := os.Stat(path2)
info1, err1 := env.Stat(path1)
info2, err2 := env.Stat(path2)
// both files exist => check using os.SameFile
// both files exist => check using env.SameFile
// the result is always authorative
if err1 == nil && err2 == nil {
same = os.SameFile(info1, info2)
same = env.SameFile(info1, info2)
authorative = true
return
}
@ -60,15 +61,15 @@ func couldBeSameFile(path1, path2 string) (same, authorative bool) {
}
// only 1 file does not exist => they could be different
if os.IsNotExist(err1) != os.IsNotExist(err2) {
if environment.IsNotExist(err1) != environment.IsNotExist(err2) {
return
}
}
{
// resolve paths absolutely
rpath1, err1 := filepath.Abs(path1)
rpath2, err2 := filepath.Abs(path2)
rpath1, err1 := env.Abs(path1)
rpath2, err2 := env.Abs(path2)
// if either path could not be resolved absolutely
// fallback to just using clean!

15
pkg/fsx/touch.go
View file

@ -1,20 +1,21 @@
package fsx
import (
"os"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Touch touches a file.
// It is similar to the unix 'touch' command.
//
// If the file does not exist exists, it is created using [os.Create].
// If the file does not exist exists, it is created using [env.Create].
// If the file does exist, it's access and modification times are updated to the current time.
func Touch(path string) error {
_, err := os.Stat(path)
func Touch(env environment.Environment, path string) error {
_, err := env.Stat(path)
switch {
case os.IsNotExist(err):
f, err := os.Create(path)
case environment.IsNotExist(err):
f, err := env.Create(path, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -24,6 +25,6 @@ func Touch(path string) error {
return err
default:
now := time.Now().Local()
return os.Chtimes(path, now, now)
return env.Chtimes(path, now, now)
}
}

22
pkg/fsx/type.go
View file

@ -2,29 +2,31 @@
package fsx
import (
"os"
"io/fs"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Exists checks if the given path exists
func Exists(path string) bool {
_, err := os.Lstat(path)
func Exists(env environment.Environment, path string) bool {
_, err := env.Lstat(path)
return err == nil
}
// IsDirectory checks if the provided path exists and is a directory
func IsDirectory(path string) bool {
info, err := os.Stat(path)
func IsDirectory(env environment.Environment, path string) bool {
info, err := env.Stat(path)
return err == nil && info.Mode().IsDir()
}
// IsFile checks if the provided path exists and is a regular file
func IsFile(path string) bool {
info, err := os.Stat(path)
func IsFile(env environment.Environment, path string) bool {
info, err := env.Stat(path)
return err == nil && info.Mode().IsRegular()
}
// IsLink checks if the provided path exists and is a symlink
func IsLink(path string) bool {
info, err := os.Lstat(path)
return err == nil && info.Mode()&os.ModeSymlink != 0
func IsLink(env environment.Environment, path string) bool {
info, err := env.Lstat(path)
return err == nil && info.Mode()&fs.ModeSymlink != 0
}

4
pkg/hostname/hostname.go
View file

@ -4,12 +4,14 @@ package hostname
import (
"os"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/Showmax/go-fqdn"
)
// FQDN attempts to return the fully qualified domain name of the host system.
// If an error occurs, may fall back to the empty string.
func FQDN() string {
func FQDN(env environment.Environment) string {
// TODO: Pass this through!
// try the hostname function
{

9
pkg/stringparser/parse.go
View file

@ -4,11 +4,12 @@ import (
"reflect"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/pkg/errors"
)
// Parse parses the provided value with the parser.
func Parse(name, value string, vField reflect.Value) error {
func Parse(env environment.Environment, name, value string, vField reflect.Value) error {
// use the validator
parser, ok := knownParsers[strings.ToLower(name)]
@ -17,7 +18,7 @@ func Parse(name, value string, vField reflect.Value) error {
}
// get the parsed value
checked, err := parser(value)
checked, err := parser(env, value)
if err != nil {
return errors.Wrapf(err, "parser %s returned error", name)
}
@ -53,8 +54,8 @@ var knownParsers map[string]Parser[any] = map[string]Parser[any]{
}
func asGenericParser[T any](parser Parser[T]) Parser[any] {
return func(s string) (value any, err error) {
value, err = parser(s)
return func(env environment.Environment, s string) (value any, err error) {
value, err = parser(env, s)
return
}
}

25
pkg/stringparser/stringparser.go
View file

@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
)
@ -17,19 +18,19 @@ import (
// Parsers can be found in this package as functions called Parse*.
// They are refered to by their name, e.g. ParseNonempty can be refered to by the name 'Nonempty'.
// See [Parse].
type Parser[T any] func(s string) (T, error)
type Parser[T any] func(env environment.Environment, s string) (T, error)
// ParseAbspath checks that s is an absolute path and returns it as-is
func ParseAbspath(s string) (string, error) {
if !fsx.IsDirectory(s) {
func ParseAbspath(env environment.Environment, s string) (string, error) {
if !fsx.IsDirectory(env, s) {
return "", errors.Errorf("%q does not exist or is not a directory", s)
}
return s, nil
}
// ParseFile checks that s is a valid file and returns it as-is
func ParseFile(s string) (string, error) {
if !fsx.IsFile(s) {
func ParseFile(env environment.Environment, s string) (string, error) {
if !fsx.IsFile(env, s) {
return "", errors.Errorf("%q does not exist or is not a regular file", s)
}
return s, nil
@ -38,7 +39,7 @@ func ParseFile(s string) (string, error) {
var errEmptyString = errors.New("value is empty")
// ParseNonEmpty checks that s is a non-empty string and returns it as-is
func ParseNonEmpty(s string) (string, error) {
func ParseNonEmpty(env environment.Environment, s string) (string, error) {
if s == "" {
return "", errEmptyString
}
@ -48,7 +49,7 @@ func ParseNonEmpty(s string) (string, error) {
var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseValidDomain checks that s is a valid domain and returns it in lowercase
func ParseValidDomain(s string) (string, error) {
func ParseValidDomain(env environment.Environment, s string) (string, error) {
if !regexpDomain.MatchString(s) {
return "", errors.Errorf("%q is not a valid domain", s)
}
@ -56,7 +57,7 @@ func ParseValidDomain(s string) (string, error) {
}
// ParseValidDomains checks that s is a comma-seperated list of valid domains and returns them in lower case
func ParseValidDomains(s string) ([]string, error) {
func ParseValidDomains(env environment.Environment, s string) ([]string, error) {
if len(s) == 0 {
return []string{}, nil
}
@ -70,13 +71,13 @@ func ParseValidDomains(s string) ([]string, error) {
}
// ParseNumber parses s as a decimal integer
func ParseNumber(s string) (int, error) {
func ParseNumber(env environment.Environment, s string) (int, error) {
value, err := strconv.ParseInt(s, 10, 64)
return int(value), err
}
// ParseHttpsURL parses a string into a url that starts with 'https://'
func ParseHttpsURL(s string) (*url.URL, error) {
func ParseHttpsURL(env environment.Environment, s string) (*url.URL, error) {
url, err := url.Parse(s)
if err != nil {
return nil, errors.Wrapf(err, "%q is not a valid URL", s)
@ -90,7 +91,7 @@ func ParseHttpsURL(s string) (*url.URL, error) {
var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseEmail checks that s represents an email, and then returns it as is.
func ParseEmail(s string) (string, error) {
func ParseEmail(env environment.Environment, s string) (string, error) {
if s == "" { // no email provided
return "", nil
}
@ -103,7 +104,7 @@ func ParseEmail(s string) (string, error) {
var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseSlug parses s as a slug and returns it as is.
func ParseSlug(s string) (string, error) {
func ParseSlug(env environment.Environment, s string) (string, error) {
if !regexpSlug.MatchString(s) {
return "", errors.Errorf("%q is not a valid slug", s)
}

11
pkg/targz/targz.go
View file

@ -6,17 +6,18 @@ import (
"compress/gzip"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Package packages the source directory into a 'tar.gz' file into destination.
// If the destination already exists, it is truncated.
//
// onCopy, when not nil, is called for each file being copied into the archive.
func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
func Package(env environment.Environment, dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
// create the target archive
archive, err := os.Create(dst)
archive, err := env.Create(dst, environment.DefaultFilePerm)
if err != nil {
return 0, err
}
@ -31,7 +32,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64,
defer tarHandle.Close()
// and walk through it!
err = filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {
err = env.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
@ -74,7 +75,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64,
}
// open the file
handle, err := os.Open(path)
handle, err := env.Open(path)
if err != nil {
return err
}

27
pkg/unpack/resource.go
View file

@ -3,9 +3,9 @@ package unpack
import (
"io"
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/pkg/errors"
)
@ -16,9 +16,9 @@ var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a
//
// onInstallFile is called for each file or directory being installed.
//
// If the destination path does not exist, it is created using [os.MakeDirs]
// If the destination path does not exist, it is created using [environment.MakeDirs]
// The directory is installed recursively.
func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
func InstallDir(env environment.Environment, dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the source file
srcFile, err := fsys.Open(src)
if err != nil {
@ -43,14 +43,14 @@ func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src
// do the installation of the directory.
// the type cast should be safe.
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
return installDir(env, dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
// installResource installs the resource at src within fsys to dst.
//
// OnInstallFile is called for each source and destination file.
// OnInstallFile may be nil.
func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
func installResource(env environment.Environment, dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the srcFile
srcFile, err := fsys.Open(src)
if err != nil {
@ -71,19 +71,19 @@ func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst,
// this is a directory, so the cast is safe!
if srcInfo.IsDir() {
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
return installDir(env, dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
// this is a regular file!
return installFile(dst, srcInfo, srcFile)
return installFile(env, dst, srcInfo, srcFile)
}
func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
func installDir(env environment.Environment, dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// create the destination
dstStat, dstErr := os.Stat(dst)
dstStat, dstErr := env.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
case environment.IsNotExist(dstErr):
if err := env.MkdirAll(dst, srcInfo.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
@ -105,6 +105,7 @@ func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src str
// iterate over all the children
for _, entry := range entries {
if err := installResource(
env,
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
fsys,
@ -117,9 +118,9 @@ func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src str
return nil
}
func installFile(dst string, srcInfo fs.FileInfo, src fs.File) error {
func installFile(env environment.Environment, dst string, srcInfo fs.FileInfo, src fs.File) error {
// create the file using the right mode!
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
file, err := env.Create(dst, srcInfo.Mode())
if err != nil {
return err
}

6
pkg/unpack/template.go
View file

@ -6,9 +6,9 @@ import (
"fmt"
"io"
"io/fs"
"os"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
@ -201,7 +201,7 @@ parseloop:
// Any existing file is truncated and overwritten.
//
// See [WriteTemplate] for possible errors.
func InstallTemplate(dst string, context map[string]string, src string, fsys fs.FS) error {
func InstallTemplate(env environment.Environment, dst string, context map[string]string, src string, fsys fs.FS) error {
// open the srcFile
srcFile, err := fsys.Open(src)
@ -222,7 +222,7 @@ func InstallTemplate(dst string, context map[string]string, src string, fsys fs.
}
// open the destination file
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
file, err := env.Create(dst, srcInfo.Mode())
if err != nil {
return err
}

2
program.go
View file

@ -41,7 +41,7 @@ func NewProgram() Program {
// when not running inside docker and we need a distillery
// then we should warn if we are not using the distillery executable.
if dis := context.Environment; !context.Args.Flags.InternalInDocker && context.Description.Requirements.NeedsDistillery && !dis.Config.UsingDistilleryExecutable() {
if dis := context.Environment; !context.Args.Flags.InternalInDocker && context.Description.Requirements.NeedsDistillery && !dis.Config.UsingDistilleryExecutable(dis.Environment) {
context.EPrintf(warnNoDeployWdcli, core.Executable, dis.Config.ExecutablePath())
}