rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'environment' package

Browse source 
This commit adds a new environment package that manages all calls to the
underlying operating system.
This commit is contained in:
Tom Wiesing 2022-09-18 14:24:22 +02:00
parent 822c70cd69
commit f19619ef9f
No known key found for this signature in database
60 changed files with 539 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/component/backup.go
View file

@ -28,8 +28,8 @@ type BackupContext interface {
// It then allows op to fill the file.
AddDirectory(path string, op func() error) error
// CopyFile copies a file from source to dst.
CopyFile(dest, src string) error
// CopyFile copies a file from src to dst.
CopyFile(dst, src string) error
// AddFile creates a new file at the provided path inside the destination.
// Passing the empty path creates the destination as a file.

16
internal/component/component.go
View file

@ -3,6 +3,7 @@ package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Component represents a logical subsystem of the distillery.
@ -38,7 +39,7 @@ type Installable interface {
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call [ComponentBase.MakeStack]
Stack() StackWithResources
Stack(env environment.Environment) StackWithResources
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
@ -47,8 +48,14 @@ type Installable interface {
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
Config *config.Config // Config is the configuration of the underlying distillery
Core // the core of the associated distillery
Dir string // Dir is the directory this component lives in
}
// Core represents the core of a distillery
type Core struct {
Environment environment.Environment // environment to use for reading / writing to and from the distillery
Config *config.Config // the configuration of the distillery
}
// Base returns a reference to the ComponentBase
@ -67,7 +74,8 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext {
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources {
func (cb ComponentBase) MakeStack(env environment.Environment, stack StackWithResources) StackWithResources {
stack.Env = env
stack.Dir = cb.Dir
return stack
}

7
internal/component/control/control.go
View file

@ -6,6 +6,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Control represents the control server
@ -24,8 +25,8 @@ func (control Control) Name() string {
//go:embed all:control control.env
var resources embed.FS
func (control Control) Stack() component.StackWithResources {
return control.ComponentBase.MakeStack(component.StackWithResources{
func (control Control) Stack(env environment.Environment) component.StackWithResources {
return control.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
@ -49,6 +50,6 @@ func (control Control) Stack() component.StackWithResources {
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: control.Config.CurrentExecutable(),
core.Executable: control.Config.CurrentExecutable(control.Environment), // TODO: Does this make sense?
}
}

3
internal/component/control/resolver.go
View file

@ -2,7 +2,6 @@ package control
import (
"fmt"
"os"
"path/filepath"
"regexp"
@ -37,7 +36,7 @@ func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler,
// open the prefix file
prefixFile := control.ResolverConfigPath()
fs, err := os.Open(prefixFile)
fs, err := control.Environment.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err

3
internal/component/control/self.go
View file

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"github.com/tkw1536/goprogram/stream"
@ -13,7 +12,7 @@ import (
// self returns the handler for the self overrides
func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(control.Config.SelfOverridesFile)
overrides, err := control.Environment.Open(control.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
if err != nil {
return redirect, err

16
internal/component/installable.go
View file

@ -2,9 +2,9 @@ package component
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/pkg/errors"
@ -29,7 +29,7 @@ type StackWithResources struct {
CopyContextFiles []string // Files to copy from the installation context
MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
MakeDirs []string // directories to ensure that exist
TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
@ -42,10 +42,11 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(
env,
is.Dir,
is.ContextPath,
is.Resources,
@ -62,6 +63,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
if err := unpack.InstallTemplate(
env,
envDest,
is.EnvContext,
is.EnvPath,
@ -78,9 +80,9 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
io.Printf("[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = fs.ModeDir
is.MakeDirsPerm = environment.DefaultDirPerm
}
if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
return err
}
}
@ -98,7 +100,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
// copy over file from context
io.Printf("[copy] %s (from %s)\n", dst, src)
if err := fsx.CopyFile(dst, src); err != nil {
if err := fsx.CopyFile(env, dst, src); err != nil {
return errors.Wrapf(err, "Unable to copy file %s", src)
}
}
@ -109,7 +111,7 @@ func (is StackWithResources) Install(io stream.IOStream, context InstallationCon
dst := filepath.Join(is.Dir, name)
io.Printf("[touch] %s\n", dst)
if err := fsx.Touch(dst); err != nil {
if err := fsx.Touch(env, dst); err != nil {
return err
}
}

4
internal/component/instances/wisski_create.go
View file

@ -21,7 +21,7 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
wisski.instances = instances
// make sure that the slug is valid!
slug, err = stringparser.ParseSlug(slug)
slug, err = stringparser.ParseSlug(instances.Environment, slug)
if err != nil {
return wisski, errInvalidSlug
}
@ -70,7 +70,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Barrel()
if err := st.Install(io, component.InstallationContext{}); err != nil {
if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
return err
}

5
internal/component/instances/wisski_pathbuilders.go
View file

@ -2,12 +2,11 @@ package instances
import (
"fmt"
"io/fs"
"os"
"path/filepath"
_ "embed"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
@ -51,7 +50,7 @@ func (wisski *WissKI) ExportPathbuilders(dest string) error {
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
if err := environment.WriteFile(wisski.instances.Core.Environment, name, pbxml, environment.DefaultFilePerm); err != nil {
return err
}
}

7
internal/component/instances/wisski_prefix.go
View file

@ -3,7 +3,6 @@ package instances
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
@ -14,7 +13,7 @@ import (
// NoPrefix checks if this WissKI instance is excluded from generating prefixes.
// TODO: Move this to the database!
func (wisski *WissKI) NoPrefix() bool {
return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
@ -41,8 +40,8 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) {
// custom prefixes
prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if fsx.IsFile(wisski.instances.Environment, prefixPath) {
prefix, err := wisski.instances.Core.Environment.Open(prefixPath)
if err != nil {
return "", err
}

6
internal/component/instances/wisski_stack.go
View file

@ -2,7 +2,6 @@ package instances
import (
"embed"
"io/fs"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component"
@ -16,6 +15,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: barrelResources,
@ -35,8 +35,7 @@ func (wisski WissKI) Barrel() component.StackWithResources {
"GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
@ -52,6 +51,7 @@ func (wisski WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
Env: wisski.instances.Environment,
},
Resources: reserveResources,

2
internal/component/sql/backup.go
View file

@ -17,7 +17,7 @@ func (*SQL) BackupName() string {
func (sql *SQL) Backup(context component.BackupContext) error {
return context.AddFile("", func(file io.Writer) error {
io := context.IO().Streams(file, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}

5
internal/component/sql/database.go
View file

@ -21,6 +21,7 @@ func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, err
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
DefaultStringSize: 256,
}
// TODO: Use sql.Core.Environment.Dial
db, err := gorm.Open(mysql.New(cfg), config)
if err != nil {
@ -63,7 +64,7 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error {
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}
@ -75,7 +76,7 @@ func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) err
// OpenShell executes a mysql shell command
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)
return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
}
// WaitShell waits for the sql database to be reachable via a docker-compose shell

8
internal/component/sql/sql.go
View file

@ -3,10 +3,10 @@ package sql
import (
"context"
"embed"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SQL struct {
@ -25,12 +25,12 @@ func (SQL) Name() string {
//go:embed all:sql
var resources embed.FS
func (ssh SQL) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SQL) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "sql",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirsPerm: environment.DefaultDirPerm,
MakeDirs: []string{
"data",
},

5
internal/component/ssh/ssh.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type SSH struct {
@ -17,8 +18,8 @@ func (SSH) Name() string {
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
func (ssh SSH) Stack(env environment.Environment) component.StackWithResources {
return ssh.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
})

11
internal/component/stack.go
View file

@ -6,7 +6,7 @@ import (
"bytes"
"errors"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
)
@ -18,6 +18,7 @@ import (
type Stack struct {
Dir string // Directory this Stack is located in
Env environment.Environment
DockerExecutable string // Path to the native docker executable to use
}
@ -101,7 +102,7 @@ func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string
code, err := ds.compose(io, compose...)
if err != nil {
return execx.ExecCommandError, nil
return environment.ExecCommandError, nil
}
return code, nil
}
@ -175,10 +176,10 @@ func (ds Stack) Down(io stream.IOStream) error {
func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
if ds.DockerExecutable == "" {
var err error
ds.DockerExecutable, err = execx.LookPathAbs("docker")
ds.DockerExecutable, err = ds.Env.LookPathAbs("docker")
if err != nil {
return execx.ExecCommandError, err
return environment.ExecCommandError, err
}
}
return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
}

11
internal/component/triplestore/database.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"mime/multipart"
"net"
"net/http"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
@ -61,6 +62,14 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
}
// create the request object
client := &http.Client{
Transport: &http.Transport{
Dial: ts.Environment.Dial,
DialTLS: func(network, addr string) (net.Conn, error) {
return nil, errors.New("not implemented")
},
},
}
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
if err != nil {
return nil, err
@ -76,7 +85,7 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
// and send it
return http.DefaultClient.Do(req)
return client.Do(req)
}
// Wait waits for the connection to the Triplestore to succeed.

7
internal/component/triplestore/triplestore.go
View file

@ -3,11 +3,11 @@ package triplestore
import (
"context"
"embed"
"io/fs"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
type Triplestore struct {
@ -26,14 +26,13 @@ func (Triplestore) Name() string {
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() component.StackWithResources {
return ts.ComponentBase.MakeStack(component.StackWithResources{
func (ts Triplestore) Stack(env environment.Environment) component.StackWithResources {
return ts.ComponentBase.MakeStack(env, component.StackWithResources{
Resources: resources,
ContextPath: "stack",
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),

15
internal/component/web/web.go
View file

@ -4,6 +4,7 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
)
// Web implements the ingress gateway for the distillery.
@ -17,11 +18,11 @@ func (Web) Name() string {
return "web"
}
func (web Web) Stack() component.StackWithResources {
func (web Web) Stack(env environment.Environment) component.StackWithResources {
if web.Config.HTTPSEnabled() {
return web.stackHTTPS()
return web.stackHTTPS(env)
} else {
return web.stackHTTP()
return web.stackHTTP(env)
}
}
@ -29,8 +30,8 @@ func (web Web) Stack() component.StackWithResources {
//go:embed web-https.env
var httpsResources embed.FS
func (web Web) stackHTTPS() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTPS(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpsResources,
ContextPath: "web-https",
EnvPath: "web-https.env",
@ -45,8 +46,8 @@ func (web Web) stackHTTPS() component.StackWithResources {
//go:embed web-http.env
var httpResources embed.FS
func (web Web) stackHTTP() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
func (web Web) stackHTTP(env environment.Environment) component.StackWithResources {
return web.MakeStack(env, component.StackWithResources{
Resources: httpResources,
ContextPath: "web-http",
EnvPath: "web-http.env",