snapshots: Prepare for restructuring

This commit renames the 'wisski' package to 'dis' and prepares the
snapshots component for restructuring.
This commit is contained in:
Tom Wiesing 2022-10-01 19:53:18 +02:00
parent f58920baf4
commit 1dac09bc03
No known key found for this signature in database
12 changed files with 62 additions and 55 deletions

View file

@ -1,51 +0,0 @@
package wisski
import (
"path/filepath"
"time"
"github.com/tkw1536/goprogram/stream"
)
// ShouldPrune determines if a file with the provided modtime
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
}
// PruneBackups prunes all backups older than the maximum backup age
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.Snapshots().ArchivePath()
// list all the files
entries, err := dis.Core.Environment.ReadDir(sPath)
if err != nil {
return err
}
for _, entry := range entries {
// skip directories
if entry.IsDir() {
continue
}
// grab info about the file
info, err := entry.Info()
if err != nil {
return err
}
// check if it should be pruned!
if !dis.ShouldPrune(info.ModTime()) {
continue
}
// assemble path, and then remove the file!
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
if err := dis.Core.Environment.Remove(path); err != nil {
return err
}
}
return nil
}

View file

@ -1,137 +0,0 @@
package wisski
import (
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/control"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/internal/component/web"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
)
// components holds the various components of the distillery
// It is inlined into the [Distillery] struct, and initialized using [makeComponent].
//
// The caller is responsible for syncronizing access across multiple goroutines.
type components struct {
// installable components
web lazy.Lazy[*web.Web]
control lazy.Lazy[*control.Control]
ssh lazy.Lazy[*ssh.SSH]
ts lazy.Lazy[*triplestore.Triplestore]
sql lazy.Lazy[*sql.SQL]
// other components
instances lazy.Lazy[*instances.Instances]
snapshots lazy.Lazy[*snapshots.Snapshots]
}
//
// Individual Components
//
func (dis *Distillery) Web() *web.Web {
return component.Initialize(dis.Core, &dis.components.web, nil)
}
func (d *Distillery) Control() *control.Control {
return component.Initialize(d.Core, &d.components.control, func(control *control.Control) {
control.ResolverFile = core.PrefixConfig
control.Instances = d.Instances()
})
}
func (dis *Distillery) SSH() *ssh.SSH {
return component.Initialize(dis.Core, &dis.components.ssh, nil)
}
func (dis *Distillery) SQL() *sql.SQL {
return component.Initialize(dis.Core, &dis.components.sql, func(sql *sql.SQL) {
sql.ServerURL = dis.Upstream.SQL
sql.PollContext = dis.Context()
sql.PollInterval = time.Second
})
}
func (dis *Distillery) Triplestore() *triplestore.Triplestore {
return component.Initialize(dis.Core, &dis.components.ts, func(ts *triplestore.Triplestore) {
ts.BaseURL = "http://" + dis.Upstream.Triplestore
ts.PollContext = dis.Context()
ts.PollInterval = time.Second
})
}
func (dis *Distillery) Instances() *instances.Instances {
return component.Initialize(dis.Core, &dis.components.instances, func(instances *instances.Instances) {
instances.SQL = dis.SQL()
instances.TS = dis.Triplestore()
})
}
func (dis *Distillery) Snapshots() *snapshots.Snapshots {
return component.Initialize(dis.Core, &dis.components.snapshots, func(snapshots *snapshots.Snapshots) {
})
}
//
// ALL COMPONENTS
//
func (dis *Distillery) Components() []component.Component {
return []component.Component{
dis.Web(),
dis.Control(),
dis.SSH(),
dis.Triplestore(),
dis.SQL(),
dis.Instances(),
}
}
//
// COMPONENT SUBTYPE GETTERS
//
// Backupable returns all the components that can be backuped up.
func (dis *Distillery) Backupable() []component.Backupable {
return getComponentSubtype[component.Backupable](dis)
}
// Installables returns all components that can be installed
func (dis *Distillery) Installables() []component.Installable {
return getComponentSubtype[component.Installable](dis)
}
// Installables returns all components that can be installed
func (dis *Distillery) Updateable() []component.Updatable {
return getComponentSubtype[component.Updatable](dis)
}
// Provisionable returns all components which can be provisioned
func (dis *Distillery) Provisionable() []component.Provisionable {
return getComponentSubtype[component.Provisionable](dis)
}
// getComponentSubtype gets all components of type T
func getComponentSubtype[T component.Component](dis *Distillery) (components []T) {
all := dis.Components()
components = make([]T, 0, len(all))
for _, c := range all {
sc, ok := c.(T)
if !ok {
continue
}
components = append(components, sc)
}
return
}

View file

@ -1,36 +0,0 @@
package wisski
import (
"context"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
// Distillery represents a WissKI Distillery
//
// It is the main structure used to interact with different components.
type Distillery struct {
// core holds the core of the distillery
component.Core
// Upstream holds information to connect to the various running
// distillery components.
//
// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
// But for now this will just hold upstream configuration.
Upstream Upstream
// components hold references to the various components of the distillery.
components
}
// Upstream contains the configuration for accessing remote configuration.
type Upstream struct {
SQL string
Triplestore string
}
// Context returns a new Context belonging to this distillery
func (dis *Distillery) Context() context.Context {
return context.Background()
}

View file

@ -1,70 +0,0 @@
package wisski
import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/exit"
)
var errNoConfigFile = exit.Error{
ExitCode: exit.ExitGeneralArguments,
Message: "Configuration File does not exist",
}
var errOpenConfig = exit.Error{
ExitCode: exit.ExitGeneralArguments,
Message: "error loading configuration file: %s",
}
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
Core: component.Core{
Environment: environment.Native{},
},
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
},
}
// we are within the docker
//
// so setup the ports to connect everything to peroperly.
// also override some of the parameters for the environment.
if flags.InternalInDocker {
dis.Upstream.SQL = "sql:3306"
dis.Upstream.Triplestore = "triplestore:7200"
params.ConfigPath = dis.Core.Environment.GetEnv("CONFIG_PATH")
}
// if we don't need to load the config, there is nothing to do
if !req.NeedsDistillery {
return
}
// try to find the configuration file
cfg := flags.ConfigPath // command line flags first
if cfg == "" {
cfg = params.ConfigPath // then globally provided files
}
if cfg == "" {
return nil, errNoConfigFile
}
// open the config file!
f, err := dis.Core.Environment.Open(params.ConfigPath)
if err != nil {
return nil, errOpenConfig.WithMessageF(err)
}
defer f.Close()
// unmarshal the config
dis.Config = &config.Config{
ConfigPath: cfg,
}
err = dis.Config.Unmarshal(dis.Core.Environment, f)
return
}

View file

@ -1,301 +0,0 @@
package wisski
import (
"encoding/json"
"fmt"
"io"
"path/filepath"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
)
// SnapshotDescription is a description for a snapshot
type SnapshotDescription struct {
Dest string // destination path
Log bool // should we log the creation of this snapshot?
Keepalive bool // should we keep the instance alive while making the snapshot?
}
// Snapshot represents the result of generating a snapshot
type Snapshot struct {
Description SnapshotDescription
Instance models.Instance
// Start and End Time of the snapshot
StartTime time.Time
EndTime time.Time
// Generic Panic that may have occured
ErrPanic interface{}
// Errors during starting and stopping the system
ErrStart error
ErrStop error
// List of files included
Manifest []string
// Errors during other parts
ErrBookkeep error
ErrPathbuilder error
ErrFilesystem error
ErrTriplestore error
ErrSQL error
}
func (snapshot Snapshot) String() string {
var builder strings.Builder
snapshot.Report(&builder)
return builder.String()
}
// Report writes a report from snapshot into w
func (snapshot Snapshot) Report(w io.Writer) (int, error) {
ww := countwriter.NewCountWriter(w)
// TODO: Errors of the writer!
encoder := json.NewEncoder(ww)
encoder.SetIndent("", " ")
io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
fmt.Fprintf(ww, "Slug: %s\n", snapshot.Instance.Slug)
fmt.Fprintf(ww, "Dest: %s\n", snapshot.Description.Dest)
fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
fmt.Fprintf(ww, "End: %s\n", snapshot.EndTime)
io.WriteString(ww, "\n")
io.WriteString(ww, "======= Description =======\n")
encoder.Encode(snapshot.Description)
io.WriteString(ww, "\n")
io.WriteString(ww, "======= Instance =======\n")
encoder.Encode(snapshot.Instance)
io.WriteString(ww, "\n")
io.WriteString(ww, "======= Errors =======\n")
fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic)
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart)
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop)
fmt.Fprintf(ww, "Bookkeep: %s\n", snapshot.ErrBookkeep)
fmt.Fprintf(ww, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
fmt.Fprintf(ww, "Filesystem: %s\n", snapshot.ErrFilesystem)
fmt.Fprintf(ww, "Triplestore: %s\n", snapshot.ErrTriplestore)
fmt.Fprintf(ww, "SQL: %s\n", snapshot.ErrSQL)
io.WriteString(ww, "\n")
io.WriteString(ww, "======= Manifest =======\n")
for _, file := range snapshot.Manifest {
io.WriteString(ww, file+"\n")
}
io.WriteString(ww, "\n")
io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
return ww.Sum()
}
// Snapshot creates a new snapshot of this instance into dest
func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
// setup the snapshot
snapshot.Description = desc
snapshot.Instance = instance.Instance
// capture anything critical, and write the end time
defer func() {
snapshot.ErrPanic = recover()
}()
// do the create keeping track of time!
logging.LogOperation(func() error {
snapshot.StartTime = time.Now().UTC()
snapshot.makeBlackbox(io, dis, instance)
snapshot.makeWhitebox(io, dis, instance)
snapshot.EndTime = time.Now().UTC()
return nil
}, io, "Writing snapshot files")
slices.Sort(snapshot.Manifest)
return
}
// makeBlackbox runs the blackbox backup of the system.
// It pauses the Instance, if a consistent state is required.
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, instance instances.WissKI) {
stack := instance.Barrel()
og := opgroup.NewOpGroup[string](4)
st := status.NewWithCompat(io.Stdout, 0)
st.Start()
defer st.Stop()
// stop the instance (unless it was explicitly asked to not do so!)
if !snapshot.Description.Keepalive {
logging.LogMessage(io, "Stopping instance")
snapshot.ErrStop = stack.Down(io)
defer func() {
logging.LogMessage(io, "Starting instance")
snapshot.ErrStart = stack.Up(io)
}()
}
// write bookkeeping information
og.GoErr(func(files chan<- string) error {
line := st.OpenLine("[snapshot bookkeeping]: ", "")
defer line.Close()
defer fmt.Fprintln(line, "done")
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
fmt.Fprintln(line, bkPath)
files <- bkPath
info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer info.Close()
// print whatever is in the database
// TODO: This should be sql code, maybe gorm can do that?
_, err = fmt.Fprintf(info, "%#v\n", instance.Instance)
return err
}, &snapshot.ErrBookkeep)
// backup the filesystem
og.GoErr(func(files chan<- string) error {
line := st.OpenLine("[snapshot filesystem]: ", "")
defer line.Close()
defer fmt.Fprintln(line, "done")
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
// copy over whatever is in the base directory
defer fmt.Fprintln(line, "done")
return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
fmt.Fprintln(line, dst)
files <- dst
})
}, &snapshot.ErrFilesystem)
// backup the graph db repository
og.GoErr(func(files chan<- string) error {
line := st.OpenLine("[snapshot triplestore]: ", "")
defer line.Close()
defer fmt.Fprintln(line, "done")
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
fmt.Fprintln(line, tsPath)
files <- tsPath
nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer nquads.Close()
// directly store the result
_, err = dis.Triplestore().Snapshot(nquads, instance.GraphDBRepository)
return err
}, &snapshot.ErrTriplestore)
// backup the sql database
og.GoErr(func(files chan<- string) error {
line := st.OpenLine("[snapshot sql]: ", "")
defer line.Close()
defer fmt.Fprintln(line, "done")
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
fmt.Fprintln(line, sqlPath)
files <- sqlPath
sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer sql.Close()
// directly store the result
return dis.SQL().Snapshot(io, sql, instance.SqlDatabase)
}, &snapshot.ErrSQL)
// wait for the group!
snapshot.waitGroup(io, og)
}
// makeWhitebox runs the whitebox backup of the system.
// The instance should be running during this step.
func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, instance instances.WissKI) {
og := opgroup.NewOpGroup[string](1)
// write pathbuilders
og.GoErr(func(files chan<- string) error {
pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
files <- pbPath
// create the directory!
if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
return err
}
// put in all the pathbuilders
return instance.ExportPathbuilders(pbPath)
}, &snapshot.ErrPathbuilder)
// wait for the group!
snapshot.waitGroup(io, og)
}
// waitGroup waits for the
func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[string]) {
// wait for the messages to return
for file := range og.Wait() {
// get the relative path to the root of the manifest.
// nothing *should* go wrong, but in case it does, use the original path.
path, err := filepath.Rel(snapshot.Description.Dest, file)
if err != nil {
path = file
}
// add the manifest
snapshot.Manifest = append(snapshot.Manifest, path)
}
}
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
stream.Println(reportPath)
// create the report file!
report, err := env.Create(reportPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = io.WriteString(report, snapshot.String())
return err
}, stream, "Writing snapshot report")
}