rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'environment' package

Browse source 
This commit adds a new environment package that manages all calls to the
underlying operating system.
This commit is contained in:
Tom Wiesing 2022-09-18 14:24:22 +02:00
parent 822c70cd69
commit f19619ef9f
No known key found for this signature in database
60 changed files with 539 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

5
internal/wisski/backup_prune.go
View file

@ -1,7 +1,6 @@
package wisski
import (
"os"
"path/filepath"
"time"
@ -18,7 +17,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.SnapshotsArchivePath()
// list all the files
entries, err := os.ReadDir(sPath)
entries, err := dis.Core.Environment.ReadDir(sPath)
if err != nil {
return err
}
@ -44,7 +43,7 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
if err := os.Remove(path); err != nil {
if err := dis.Core.Environment.Remove(path); err != nil {
return err
}
}

2
internal/wisski/component.go
View file

@ -58,7 +58,7 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
}
base := c.Base()
base.Config = dis.Config
base.Core = dis.Core
if base.Dir == "" {
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", c.Name())
}

9
internal/wisski/distillery.go
View file

@ -3,16 +3,15 @@ package wisski
import (
"context"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
// Distillery represents a WissKI Distillery
//
// It is the main structure used to interact with different components.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// core holds the core of the distillery
component.Core
// Upstream holds information to connect to the various running
// distillery components.
@ -25,7 +24,7 @@ type Distillery struct {
components
}
// Upstream are the upstream urls connecting to the various external components.
// Upstream contains the configuration for accessing remote configuration.
type Upstream struct {
SQL string
Triplestore string

13
internal/wisski/init.go
View file

@ -1,10 +1,10 @@
package wisski
import (
"os"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
)
@ -21,6 +21,9 @@ var errOpenConfig = exit.Error{
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
Core: component.Core{
Environment: environment.Native{},
},
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
@ -34,7 +37,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
if flags.InternalInDocker {
dis.Upstream.SQL = "sql:3306"
dis.Upstream.Triplestore = "triplestore:7200"
params.ConfigPath = os.Getenv("CONFIG_PATH")
params.ConfigPath = dis.Core.Environment.GetEnv("CONFIG_PATH")
}
// if we don't need to load the config, there is nothing to do
@ -52,7 +55,7 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
}
// open the config file!
f, err := os.Open(params.ConfigPath)
f, err := dis.Core.Environment.Open(params.ConfigPath)
if err != nil {
return nil, errOpenConfig.WithMessageF(err)
}
@ -62,6 +65,6 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
dis.Config = &config.Config{
ConfigPath: cfg,
}
err = dis.Config.Unmarshal(f)
err = dis.Config.Unmarshal(dis.Core.Environment, f)
return
}

29
internal/wisski/snapshot.go
View file

@ -4,8 +4,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
@ -13,6 +11,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
@ -42,7 +41,7 @@ func (dis *Distillery) SnapshotsArchivePath() string {
// The path is guaranteed to not exist.
func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
// TODO: Consider moving these into a subdirectory with the provided prefix.
for path == "" || fsx.Exists(path) {
for path == "" || fsx.Exists(dis.Environment, path) {
name := dis.newSnapshotName(prefix) + ".tar.gz"
path = filepath.Join(dis.SnapshotsArchivePath(), name)
}
@ -64,9 +63,9 @@ func (*Distillery) newSnapshotName(prefix string) string {
// NewSnapshotStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
for path == "" || os.IsExist(err) {
for path == "" || environment.IsExist(err) {
path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
err = os.Mkdir(path, os.ModeDir)
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
}
if err != nil {
path = ""
@ -210,7 +209,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
files <- bkPath
info, err := os.Create(bkPath)
info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -227,7 +226,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
// copy over whatever is in the base directory
return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
files <- dst
})
}, &snapshot.ErrFilesystem)
@ -237,7 +236,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
files <- tsPath
nquads, err := os.Create(tsPath)
nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -253,7 +252,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
files <- sqlPath
sql, err := os.Create(sqlPath)
sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -279,7 +278,7 @@ func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, inst
files <- pbPath
// create the directory!
if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
return err
}
@ -312,20 +311,20 @@ func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[stri
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
io.Println(reportPath)
stream.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = report.WriteString(snapshot.String())
_, err = io.WriteString(report, snapshot.String())
return err
}, io, "Writing snapshot report")
}, stream, "Writing snapshot report")
}