rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'environment' package

Browse source 
This commit adds a new environment package that manages all calls to the
underlying operating system.
This commit is contained in:
Tom Wiesing 2022-09-18 14:24:22 +02:00
parent 822c70cd69
commit f19619ef9f
No known key found for this signature in database
60 changed files with 539 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

8
internal/backup/backup.go
View file

@ -2,14 +2,13 @@
package backup
import (
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
@ -61,6 +60,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
err: bc.Backup(context),
}
}(bc, &context{
env: dis.Core.Environment,
io: io,
dst: filepath.Join(backup.Description.Dest, bc.BackupName()),
files: files,
@ -73,7 +73,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
defer wg.Done()
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(instancesBackupDir, environment.DefaultDirPerm); err != nil {
backup.InstanceListErr = err
return
}
@ -89,7 +89,7 @@ func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
for i, instance := range instances {
backup.InstanceSnapshots[i] = func() wisski.Snapshot {
dir := filepath.Join(instancesBackupDir, instance.Slug)
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil {
return wisski.Snapshot{
ErrPanic: err,
}

10
internal/backup/context.go
View file

@ -3,16 +3,16 @@ package backup
import (
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/stream"
)
// context implements [components.BackupContext]
type context struct {
env environment.Environment
io stream.IOStream
dst string // destination directory
files chan string // files channel
@ -54,7 +54,7 @@ func (bc *context) AddDirectory(path string, op func() error) error {
}
// run the make directory
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
if err := bc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil {
return err
}
@ -72,7 +72,7 @@ func (bc *context) CopyFile(dst, src string) error {
return err
}
bc.sendPath(dst)
return fsx.CopyFile(dstPath, src)
return fsx.CopyFile(bc.env, dstPath, src)
}
func (bc *context) AddFile(path string, op func(file io.Writer) error) error {
@ -83,7 +83,7 @@ func (bc *context) AddFile(path string, op func(file io.Writer) error) error {
}
// create the file
file, err := os.Create(dst)
file, err := bc.env.Create(dst, environment.DefaultFilePerm)
if err != nil {
return err
}

12
internal/backup/report.go
View file

@ -4,13 +4,13 @@ import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/stream"
)
@ -96,20 +96,20 @@ func (backup Backup) Report(w io.Writer) (int, error) {
// WriteReport writes out the report belonging to this backup.
// It is a separate function, to allow writing it indepenently of the rest.
func (backup Backup) WriteReport(io stream.IOStream) error {
func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
io.Println(reportPath)
stream.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = report.WriteString(backup.String())
_, err = io.WriteString(report, backup.String())
return err
}, io, "Writing backup report")
}, stream, "Writing backup report")
}

4
internal/component/backup.go
View file

@ -28,8 +28,8 @@ type BackupContext interface {
// It then allows op to fill the file.
AddDirectory(path string, op func() error) error
// CopyFile copies a file from source to dst.
CopyFile(dest, src string) error
// CopyFile copies a file from src to dst.
CopyFile(dst, src string) error
// AddFile creates a new file at the provided path inside the destination.
// Passing the empty path creates the destination as a file.

16
internal/component/component.go
View file

@ -3,6 +3,7 @@ package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Component represents a logical subsystem of the distillery.
@ -38,7 +39,7 @@ type Installable interface {
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call [ComponentBase.MakeStack]
Stack() StackWithResources
Stack(env environment.Environment) StackWithResources
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
@ -47,8 +48,14 @@ type Installable interface {
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
Config *config.Config // Config is the configuration of the underlying distillery
Core // the core of the associated distillery
Dir string // Dir is the directory this component lives in
}
// Core represents the core of a distillery
type Core struct {
Environment environment.Environment // environment to use for reading / writing to and from the distillery
Config *config.Config // the configuration of the distillery
}
// Base returns a reference to the ComponentBase
@ -67,7 +74,8 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext {
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources {
func (cb ComponentBase) MakeStack(env environment.Environment, stack StackWithResources) StackWithResources {
stack.Env = env
stack.Dir = cb.Dir
return stack
}

7
internal/component/control/control.go
View file

@ -6,6 +6,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Control represents the control server
@ -24,8 +25,8 @@ func (control Control) Name() string {
//go:embed all:control control.env
var resources embed.FS
func (control Control) Stack() component.StackWithResources {
return control.ComponentBase.MakeStack(component.StackWithResources{
func (control Control) Stack(env environment.Environment) component.StackWithResources {
return control.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
@ -49,6 +50,6 @@ func (control Control) Stack() component.StackWithResources {
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: control.Config.CurrentExecutable(),
core.Executable: control.Config.CurrentExecutable(control.Environment), // TODO: Does this make sense?
}
}

3
internal/component/control/resolver.go
View file

@ -2,7 +2,6 @@ package control
import (
"fmt"
"os"
"path/filepath"
"regexp"
@ -37,7 +36,7 @@ func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler,
// open the prefix file
prefixFile := control.ResolverConfigPath()
fs, err := os.Open(prefixFile)
fs, err := control.Environment.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err

3
internal/component/control/self.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"github.com/tkw1536/goprogram/stream"
@ -13,7 +12,7 @@ import (
// self returns the handler for the self overrides
func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(control.Config.SelfOverridesFile)
overrides, err := control.Environment.Open(control.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
if err != nil {
return redirect, err

16
internal/component/installable.go
View file

@ -2,9 +2,9 @@ package component
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/pkg/errors"
@ -29,7 +29,7 @@ type StackWithResources struct {
CopyContextFiles []string // Files to copy from the installation context
MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
MakeDirs []string // directories to ensure that exist
TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
@ -42,10 +42,11 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(
env,
is.Dir,
is.ContextPath,
is.Resources,
@ -62,6 +63,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
if err := unpack.InstallTemplate(
env,
envDest,
is.EnvContext,
is.EnvPath,
@ -78,9 +80,9 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
io.Printf("[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = fs.ModeDir
is.MakeDirsPerm = environment.DefaultDirPerm
}
if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
return err
}
}
@ -98,7 +100,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
// copy over file from context
io.Printf("[copy] %s (from %s)\n", dst, src)
if err := fsx.CopyFile(dst, src); err != nil {
if err := fsx.CopyFile(env, dst, src); err != nil {
return errors.Wrapf(err, "Unable to copy file %s", src)
}
}
@ -109,7 +111,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
dst := filepath.Join(is.Dir, name)
io.Printf("[touch] %s\n", dst)
if err := fsx.Touch(dst); err != nil {
if err := fsx.Touch(env, dst); err != nil {
return err
}
}

4
internal/component/instances/wisski_create.go
View file

@ -21,7 +21,7 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
wisski.instances = instances
// make sure that the slug is valid!
slug, err = stringparser.ParseSlug(slug)
slug, err = stringparser.ParseSlug(instances.Environment, slug)
if err != nil {
return wisski, errInvalidSlug
}
@ -70,7 +70,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Barrel()
if err := st.Install(io, component.InstallationContext{}); err != nil {
if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
return err
}

5
internal/component/instances/wisski_pathbuilders.go
View file

@ -2,12 +2,11 @@ package instances
import (
"fmt"
"io/fs"
"os"
"path/filepath"
_ "embed"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@ -51,7 +50,7 @@ func (wisski *WissKI) ExportPathbuilders(dest string) error {
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
if err := environment.WriteFile(wisski.instances.Core.Environment, name, pbxml, environment.DefaultFilePerm); err != nil {
return err
}
}

7
internal/component/instances/wisski_prefix.go
View file

@ -3,7 +3,6 @@ package instances
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
@ -14,7 +13,7 @@ import (
// NoPrefix checks if this WissKI instance is excluded from generating prefixes.
// TODO: Move this to the database!
func (wisski *WissKI) NoPrefix() bool {
return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
@ -41,8 +40,8 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) {
// custom prefixes
prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if fsx.IsFile(wisski.instances.Environment, prefixPath) {
prefix, err := wisski.instances.Core.Environment.Open(prefixPath)
if err != nil {
return "", err
}

6
internal/component/instances/wisski_stack.go
View file

@ -2,7 +2,6 @@ package instances
import (
"embed"
"io/fs"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component"
@ -16,6 +15,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: barrelResources,
@ -35,8 +35,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
"GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
@ -52,6 +51,7 @@ func (wisski WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: reserveResources,

2
internal/component/sql/backup.go
View file

@ -17,7 +17,7 @@ func (*SQL) BackupName() string {
func (sql *SQL) Backup(context component.BackupContext) error {
return context.AddFile("", func(file io.Writer) error {
io := context.IO().Streams(file, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}

5
internal/component/sql/database.go
View file

@ -21,6 +21,7 @@ func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, err
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
DefaultStringSize: 256,
}
// TODO: Use sql.Core.Environment.Dial
db, err := gorm.Open(mysql.New(cfg), config)
if err != nil {
@ -63,7 +64,7 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error {
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}
@ -75,7 +76,7 @@ func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) err
// OpenShell executes a mysql shell command
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)
return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
}
// WaitShell waits for the sql database to be reachable via a docker-compose shell

8
internal/component/sql/sql.go
View file

@ -3,10 +3,10 @@ package sql
import (
"context"
"embed"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SQL struct {
@ -25,12 +25,12 @@ func (SQL) Name() string {
//go:embed all:sql
var resources embed.FS
func (ssh SQL) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SQL) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "sql",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirsPerm: environment.DefaultDirPerm,
MakeDirs: []string{
"data",
},

5
internal/component/ssh/ssh.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SSH struct {
@ -17,8 +18,8 @@ func (SSH) Name() string {
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SSH) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
})

11
internal/component/stack.go
View file

@ -6,7 +6,7 @@ import (
"bytes"
"errors"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
)
@ -18,6 +18,7 @@ import (
type Stack struct {
Dir string // Directory this Stack is located in
Env environment.Environment
DockerExecutable string // Path to the native docker executable to use
}
@ -101,7 +102,7 @@ func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string
code, err := ds.compose(io, compose...)
if err != nil {
return execx.ExecCommandError, nil
return environment.ExecCommandError, nil
}
return code, nil
}
@ -175,10 +176,10 @@ func (ds Stack) Down(io stream.IOStream) error {
func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
if ds.DockerExecutable == "" {
var err error
ds.DockerExecutable, err = execx.LookPathAbs("docker")
ds.DockerExecutable, err = ds.Env.LookPathAbs("docker")
if err != nil {
return execx.ExecCommandError, err
return environment.ExecCommandError, err
}
}
return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
}

11
internal/component/triplestore/database.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
@ -61,6 +62,14 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
}
// create the request object
client := &http.Client{
Transport: &http.Transport{
Dial: ts.Environment.Dial,
DialTLS: func(network, addr string) (net.Conn, error) {
return nil, errors.New("not implemented")
},
},
}
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
if err != nil {
return nil, err
@ -76,7 +85,7 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
// and send it
return http.DefaultClient.Do(req)
return client.Do(req)
}
// Wait waits for the connection to the Triplestore to succeed.

7
internal/component/triplestore/triplestore.go
View file

@ -3,11 +3,11 @@ package triplestore
import (
"context"
"embed"
"io/fs"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type Triplestore struct {
@ -26,14 +26,13 @@ func (Triplestore) Name() string {
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() component.StackWithResources {
return ts.ComponentBase.MakeStack(component.StackWithResources{
func (ts Triplestore) Stack(env environment.Environment) component.StackWithResources {
return ts.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),

15
internal/component/web/web.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Web implements the ingress gateway for the distillery.
@ -17,11 +18,11 @@ func (Web) Name() string {
return "web"
}
func (web Web) Stack() component.StackWithResources {
func (web Web) Stack(env environment.Environment) component.StackWithResources {
if web.Config.HTTPSEnabled() {
return web.stackHTTPS()
return web.stackHTTPS(env)
} else {
return web.stackHTTP()
return web.stackHTTP(env)
}
}
@ -29,8 +30,8 @@ func (web Web) Stack() component.StackWithResources {
//go:embed web-https.env
var httpsResources embed.FS
func (web Web) stackHTTPS() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTPS(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpsResources,
ContextPath: "web-https",
EnvPath: "web-https.env",
@ -45,8 +46,8 @@ func (web Web) stackHTTPS() component.StackWithResources {
//go:embed web-http.env
var httpResources embed.FS
func (web Web) stackHTTP() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTP(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpResources,
ContextPath: "web-http",
EnvPath: "web-http.env",

14
internal/config/executable.go
View file

@ -1,10 +1,10 @@
package config
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
@ -14,19 +14,19 @@ func (cfg Config) ExecutablePath() string {
}
// UsingDistilleryExecutable checks if the current process is using the distillery executable
func (cfg Config) UsingDistilleryExecutable() bool {
exe, err := os.Executable()
func (cfg Config) UsingDistilleryExecutable(env environment.Environment) bool {
exe, err := env.Executable()
if err != nil {
return false
}
return fsx.SameFile(exe, cfg.ExecutablePath())
return fsx.SameFile(env, exe, cfg.ExecutablePath())
}
// CurrentExecutable returns the path to the current executable being used.
// When it does not exist, falls back to the default executable.
func (cfg Config) CurrentExecutable() string {
exe, err := os.Executable()
if err != nil || !fsx.IsFile(exe) {
func (cfg Config) CurrentExecutable(env environment.Environment) string {
exe, err := env.Executable()
if err != nil || !fsx.IsFile(env, exe) {
return cfg.ExecutablePath()
}
return exe

19
internal/config/read.go
View file

@ -4,6 +4,7 @@ import (
"io"
"reflect"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/envreader"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
)
@ -16,7 +17,7 @@ import (
// When a key is missing, it is set to the default value.
//
// See also [stringparser.Parse].
func (config *Config) Unmarshal(src io.Reader) error {
func (config *Config) Unmarshal(env environment.Environment, src io.Reader) error {
// read all the values!
values, err := envreader.ReadAll(src)
if err != nil {
@ -32,26 +33,26 @@ func (config *Config) Unmarshal(src io.Reader) error {
tField := tConfig.Field(i)
vField := vConfig.FieldByName(tField.Name)
env := tField.Tag.Get("env")
dflt := tField.Tag.Get("default")
parser := tField.Tag.Get("parser")
tEnv := tField.Tag.Get("env")
tDefault := tField.Tag.Get("default")
tParser := tField.Tag.Get("parser")
// skip it if it isn't loaded!
if env == "" {
if tEnv == "" {
continue
}
// read the value with a default
value, ok := values[env]
value, ok := values[tEnv]
if !ok || value == "" {
if dflt == "" {
if tDefault == "" {
continue
}
value = dflt
value = tDefault
}
// parse the value!
if err := stringparser.Parse(parser, value, vField); err != nil {
if err := stringparser.Parse(env, tParser, value, vField); err != nil {
return err
}
}

5
internal/config/template.go
View file

@ -7,6 +7,7 @@ import (
"reflect"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/hostname"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
@ -29,13 +30,13 @@ type Template struct {
}
// SetDefaults sets defaults on the template
func (tpl *Template) SetDefaults() (err error) {
func (tpl *Template) SetDefaults(env environment.Environment) (err error) {
if tpl.DeployRoot == "" {
tpl.DeployRoot = core.BaseDirectoryDefault
}
if tpl.DefaultDomain == "" {
tpl.DefaultDomain = hostname.FQDN()
tpl.DefaultDomain = hostname.FQDN(env)
}
if tpl.SelfOverridesFile == "" {

11
internal/core/meta.go
View file

@ -3,10 +3,11 @@ package core
import (
"errors"
"io/fs"
"os"
"os/user"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// MetaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable.
@ -33,7 +34,7 @@ var errReadBaseDirectoryEmpty = errors.New("ReadBaseDirectory: Directory is empt
// Use [ParamsFromEnv] to initialize parameters completely.
//
// It does not perform any reading of files.
func ReadBaseDirectory() (value string, err error) {
func ReadBaseDirectory(env environment.Environment) (value string, err error) {
// get the path!
path, err := MetaConfigPath()
if err != nil {
@ -41,7 +42,7 @@ func ReadBaseDirectory() (value string, err error) {
}
// read the meta config file!
contents, err := os.ReadFile(path)
contents, err := environment.ReadFile(env, path)
if err != nil {
return "", err
}
@ -59,7 +60,7 @@ func ReadBaseDirectory() (value string, err error) {
}
// WriteBaseDirectory writes the base directory to the environment, or returns an error
func WriteBaseDirectory(dir string) error {
func WriteBaseDirectory(env environment.Environment, dir string) error {
// get the path!
path, err := MetaConfigPath()
if err != nil {
@ -67,5 +68,5 @@ func WriteBaseDirectory(dir string) error {
}
// just put the directory inside it!
return os.WriteFile(path, []byte(dir), fs.ModePerm)
return environment.WriteFile(env, path, []byte(dir), fs.ModePerm)
}

7
internal/core/params.go
View file

@ -1,8 +1,9 @@
package core
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Params are used to initialize the excutable.
@ -15,9 +16,9 @@ type Params struct {
func ParamsFromEnv() (params Params, err error) {
// try to read the base directory!
value, err := ReadBaseDirectory()
value, err := ReadBaseDirectory(environment.Native{}) // TODO: Are we sure about the native environment here?
switch {
case os.IsNotExist(err):
case environment.IsNotExist(err):
params.ConfigPath = BaseDirectoryDefault
case err == nil:
params.ConfigPath = value

5
internal/wisski/backup_prune.go
View file

@ -1,7 +1,6 @@
package wisski
import (
"os"
"path/filepath"
"time"
@ -18,7 +17,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.SnapshotsArchivePath()
// list all the files
entries, err := os.ReadDir(sPath)
entries, err := dis.Core.Environment.ReadDir(sPath)
if err != nil {
return err
}
@ -44,7 +43,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
if err := os.Remove(path); err != nil {
if err := dis.Core.Environment.Remove(path); err != nil {
return err
}
}

2
internal/wisski/component.go
View file

@ -58,7 +58,7 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
}
base := c.Base()
base.Config = dis.Config
base.Core = dis.Core
if base.Dir == "" {
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", c.Name())
}

9
internal/wisski/distillery.go
View file

@ -3,16 +3,15 @@ package wisski
import (
"context"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
// Distillery represents a WissKI Distillery
//
// It is the main structure used to interact with different components.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// core holds the core of the distillery
component.Core
// Upstream holds information to connect to the various running
// distillery components.
@ -25,7 +24,7 @@ type Distillery struct {
components
}
// Upstream are the upstream urls connecting to the various external components.
// Upstream contains the configuration for accessing remote configuration.
type Upstream struct {
SQL string
Triplestore string

13
internal/wisski/init.go
View file

@ -1,10 +1,10 @@
package wisski
import (
"os"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
)
@ -21,6 +21,9 @@ var errOpenConfig = exit.Error{
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
Core: component.Core{
Environment: environment.Native{},
},
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
@ -34,7 +37,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
if flags.InternalInDocker {
dis.Upstream.SQL = "sql:3306"
dis.Upstream.Triplestore = "triplestore:7200"
params.ConfigPath = os.Getenv("CONFIG_PATH")
params.ConfigPath = dis.Core.Environment.GetEnv("CONFIG_PATH")
}
// if we don't need to load the config, there is nothing to do
@ -52,7 +55,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
}
// open the config file!
f, err := os.Open(params.ConfigPath)
f, err := dis.Core.Environment.Open(params.ConfigPath)
if err != nil {
return nil, errOpenConfig.WithMessageF(err)
}
@ -62,6 +65,6 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
dis.Config = &config.Config{
ConfigPath: cfg,
}
err = dis.Config.Unmarshal(f)
err = dis.Config.Unmarshal(dis.Core.Environment, f)
return
}

29
internal/wisski/snapshot.go
View file

@ -4,8 +4,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
@ -13,6 +11,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
@ -42,7 +41,7 @@ func (dis *Distillery) SnapshotsArchivePath() string {
// The path is guaranteed to not exist.
func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
// TODO: Consider moving these into a subdirectory with the provided prefix.
for path == "" || fsx.Exists(path) {
for path == "" || fsx.Exists(dis.Environment, path) {
name := dis.newSnapshotName(prefix) + ".tar.gz"
path = filepath.Join(dis.SnapshotsArchivePath(), name)
}
@ -64,9 +63,9 @@ func (*Distillery) newSnapshotName(prefix string) string {
// NewSnapshotStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
for path == "" || os.IsExist(err) {
for path == "" || environment.IsExist(err) {
path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
err = os.Mkdir(path, os.ModeDir)
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
}
if err != nil {
path = ""
@ -210,7 +209,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
files <- bkPath
info, err := os.Create(bkPath)
info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -227,7 +226,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
// copy over whatever is in the base directory
return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
files <- dst
})
}, &snapshot.ErrFilesystem)
@ -237,7 +236,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
files <- tsPath
nquads, err := os.Create(tsPath)
nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -253,7 +252,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
files <- sqlPath
sql, err := os.Create(sqlPath)
sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -279,7 +278,7 @@ func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, inst
files <- pbPath
// create the directory!
if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
return err
}
@ -312,20 +311,20 @@ func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[stri
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
io.Println(reportPath)
stream.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = report.WriteString(snapshot.String())
_, err = io.WriteString(report, snapshot.String())
return err
}, io, "Writing snapshot report")
}, stream, "Writing snapshot report")
}