snapshots: Handle as separate components

internal/component/backup.go

@@ -2,8 +2,12 @@ package component
 
 import (
 	"io"
+	"path/filepath"
 
 	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 )
 
@@ -32,6 +36,9 @@ type StagingContext interface {
 	// CopyFile copies a file from src to dst.
 	CopyFile(dst, src string) error
 
+	// CopyDirectory copies a directory from src to dst.
+	CopyDirectory(dst, src string) error
+
 	// AddFile creates a new file at the provided path inside the destination.
 	// Passing the empty path creates the destination as a file.
 	//
@@ -56,3 +63,109 @@ type Snapshotable interface {
 	// Snapshot snapshots a part of the instance
 	Snapshot(wisski models.Instance, context StagingContext) error
 }
+
+// NewStagingContext returns a new [StagingContext]
+func NewStagingContext(env environment.Environment, io stream.IOStream, path string, manifest chan<- string) StagingContext {
+	return &stagingContext{
+		env:      env,
+		io:       io,
+		path:     path,
+		manifest: manifest,
+	}
+}
+
+// stagingContext implements [components.StagingContext]
+type stagingContext struct {
+	env      environment.Environment // environment
+	io       stream.IOStream         // context the files are sent to
+	path     string                  // path to send files to
+	manifest chan<- string           // channel the manifest is sent to
+}
+
+func (bc *stagingContext) sendPath(path string) {
+	// resolve the path, or bail out!
+	// TODO: Use the relative path here!
+	dst, err := bc.resolve(path)
+	if err != nil {
+		return
+	}
+
+	bc.io.Println(dst)
+	bc.manifest <- dst
+}
+
+func (bc *stagingContext) IO() stream.IOStream {
+	return bc.io
+}
+
+var errResolveAbsolute = errors.New("resolve: path must be relative")
+
+func (bc *stagingContext) resolve(path string) (dest string, err error) {
+	if path == "" {
+		return bc.path, nil
+	}
+	if filepath.IsAbs(path) {
+		return "", errResolveAbsolute
+	}
+	return filepath.Join(bc.path, path), nil
+}
+
+func (sc *stagingContext) AddDirectory(path string, op func() error) error {
+	// resolve the path!
+	dst, err := sc.resolve(path)
+	if err != nil {
+		return err
+	}
+
+	// run the make directory
+	if err := sc.env.Mkdir(dst, environment.DefaultDirPerm); err != nil {
+		return err
+	}
+
+	// tell the files that we are creating it!
+	sc.sendPath(path)
+
+	// and run the files!
+	return op()
+}
+
+func (sc *stagingContext) CopyFile(dst, src string) error {
+	dstPath, err := sc.resolve(dst)
+	if err != nil {
+		return err
+	}
+	sc.sendPath(dst)
+	return fsx.CopyFile(sc.env, dstPath, src)
+}
+
+func (sc *stagingContext) CopyDirectory(dst, src string) error {
+	dstPath, err := sc.resolve(dst)
+	if err != nil {
+		return err
+	}
+
+	return fsx.CopyDirectory(sc.env, dstPath, src, func(dst, src string) {
+		sc.sendPath(dst)
+	})
+}
+
+func (sc *stagingContext) AddFile(path string, op func(file io.Writer) error) error {
+	// resolve the path!
+	dst, err := sc.resolve(path)
+	if err != nil {
+		return err
+	}
+
+	// create the file
+	file, err := sc.env.Create(dst, environment.DefaultFilePerm)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// tell them that we are creating it!
+	sc.sendPath(path)
+
+	// and do whatever they wanted to do
+	return op(file)
+}

internal/component/component.go

@@ -1,12 +1,6 @@
 // Package component holds the main abstraction for components.
 package component
 
-import (
-	"reflect"
-
-	"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
-)
-
 // Component represents a logical subsystem of the distillery.
 // Every component must embed [ComponentBase] and should be initialized using [Initialize].
 //
@@ -36,35 +30,3 @@ type ComponentBase struct {
 func (cb *ComponentBase) getBase() *ComponentBase {
 	return cb
 }
-
-// Initialize makes or returns a component based on a lazy.
-//
-// C is the type of component to initialize. It must be backed by a pointer, or Initialize will panic.
-//
-// dis is the distillery to initialize components for
-// field is a pointer to the appropriate struct field within the distillery components
-// init is called with a new non-nil component to initialize it.
-// It may be nil, to indicate no additional initialization is required.
-//
-// makeComponent returns the new or existing component instance
-func Initialize[C Component](core Core, field *lazy.Lazy[C], init func(C)) C {
-
-	// get the typeof C and make sure that it is a pointer type!
-	typC := reflect.TypeOf((*C)(nil)).Elem()
-	if typC.Kind() != reflect.Pointer {
-		panic("Initialize: C must be backed by a pointer")
-	}
-
-	// return the field
-	return field.Get(func() (c C) {
-		c = reflect.New(typC.Elem()).Interface().(C)
-		if init != nil {
-			init(c)
-		}
-
-		base := c.getBase()
-		base.Core = core
-
-		return
-	})
-}

internal/component/extras/bookkeeping.go

@@ -0,0 +1,29 @@
+package extras
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+)
+
+type Bookkeeping struct {
+	component.ComponentBase
+}
+
+func (Bookkeeping) Name() string { return "bookkeeping" }
+
+// SnapshotNeedsRunning returns if this Snapshotable requires a running instance.
+func (Bookkeeping) SnapshotNeedsRunning() bool { return false }
+
+// SnapshotName returns a new name to be used as an argument for path.
+func (Bookkeeping) SnapshotName() string { return "bookkeeping.txt" }
+
+// Snapshot creates a snapshot of this instance
+func (*Bookkeeping) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.AddFile(".", func(file io.Writer) error {
+		_, err := fmt.Fprintf(file, "%#v\n", wisski)
+		return err
+	})
+}

internal/component/extras/config.go

@@ -11,7 +11,7 @@ type Config struct {
 	component.ComponentBase
 }
 
-func (Config) Name() string { return "extra-config" }
+func (Config) Name() string { return "config" }
 
 func (*Config) BackupName() string {
 	return "config"

internal/component/extras/filesystem.go

@@ -0,0 +1,24 @@
+package extras
+
+import (
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+)
+
+// Filesystem implements snapshotting an instnace filesystem
+type Filesystem struct {
+	component.ComponentBase
+}
+
+func (Filesystem) Name() string { return "filesystem" }
+
+// SnapshotNeedsRunning returns if this Snapshotable requires a running instance.
+func (Filesystem) SnapshotNeedsRunning() bool { return false }
+
+// SnapshotName returns a new name to be used as an argument for path.
+func (Filesystem) SnapshotName() string { return "data" }
+
+// Snapshot creates a snapshot of this instance
+func (*Filesystem) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.CopyDirectory(".", wisski.FilesystemBase)
+}

internal/component/extras/pathbuilders.go

@@ -0,0 +1,39 @@
+package extras
+
+import (
+	"io"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+)
+
+type Pathbuilders struct {
+	component.ComponentBase
+	Instances *instances.Instances
+}
+
+func (Pathbuilders) Name() string { return "pathbuilders" }
+
+func (Pathbuilders) SnapshotNeedsRunning() bool { return true }
+
+func (Pathbuilders) SnapshotName() string { return "pathbuilders" }
+
+func (pbs *Pathbuilders) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.AddDirectory(".", func() error {
+		builders, err := pbs.Instances.Instance(wisski).AllPathbuilders()
+		if err != nil {
+			return err
+		}
+
+		for name, bytes := range builders {
+			if err := context.AddFile(name+".xml", func(file io.Writer) error {
+				_, err := file.Write([]byte(bytes))
+				return err
+			}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}

internal/component/instances/instances.go

@@ -37,6 +37,16 @@ var errSQL = exit.Error{
 	ExitCode: exit.ExitGeneric,
 }
 
+// Instance is a convenience function to return an instance based on a model slug.
+// When the instance does not exist, returns nil.
+func (instances *Instances) Instance(instance models.Instance) *WissKI {
+	i, err := instances.WissKI(instance.Slug)
+	if err != nil {
+		return nil
+	}
+	return &i
+}
+
 // WissKI returns the WissKI with the provided slug, if it exists.
 // It the WissKI does not exist, returns ErrWissKINotFound.
 func (instances *Instances) WissKI(slug string) (i WissKI, err error) {

internal/component/instances/wisski_pathbuilders.go

@@ -1,14 +1,9 @@
 package instances
 
 import (
-	"fmt"
-	"path/filepath"
-
 	_ "embed"
 
-	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/tkw1536/goprogram/stream"
-	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 )
 
@@ -34,26 +29,3 @@ func (wisski *WissKI) AllPathbuilders() (pathbuilders map[string]string, err err
 	err = wisski.ExecPHPScript(stream.FromDebug(), &pathbuilders, exportPathbuilderPHP, "all_xml")
 	return
 }
-
-// ExportPathbuilders writes pathbuilders into the directory dest
-func (wisski *WissKI) ExportPathbuilders(dest string) error {
-	pathbuilders, err := wisski.AllPathbuilders()
-	if err != nil {
-		return err
-	}
-
-	// sort the names of the pathbuilders
-	names := maps.Keys(pathbuilders)
-	slices.Sort(names)
-
-	// write each into a file!
-	for _, name := range names {
-		pbxml := []byte(pathbuilders[name])
-		name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
-		if err := environment.WriteFile(wisski.instances.Core.Environment, name, pbxml, environment.DefaultFilePerm); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

internal/component/pool.go

@@ -0,0 +1,131 @@
+package component
+
+import (
+	"reflect"
+	"sync"
+	"sync/atomic"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/rlock"
+	"github.com/tkw1536/goprogram/lib/reflectx"
+)
+
+// Pool represents a pool of components
+type Pool struct {
+	rLock rlock.RLock
+
+	// the actual queue of initi functions!
+	nested uint64 // is the q active?
+	queue  []func(thread int32)
+
+	// global initalization!
+	initOnce sync.Once
+
+	// components and lock!
+	cLock      sync.Mutex
+	components map[string]Component
+}
+
+func (p *Pool) init() {
+	p.initOnce.Do(func() {
+		p.components = make(map[string]Component)
+	})
+}
+
+// ComponentDescription describes a component
+type ComponentDescription struct {
+	Type reflect.Type
+	Elem reflect.Type
+	Name string
+}
+
+// New creates a new ComponentDescription
+func (cd ComponentDescription) New() any {
+	return reflect.New(cd.Elem).Interface()
+}
+
+// GetDescription gets the description of a component type
+func GetDescription[C Component]() (desc ComponentDescription) {
+	desc.Type = reflectx.TypeOf[C]()
+	if desc.Type.Kind() != reflect.Pointer {
+		panic("GetDescription: C must be backed by a pointer")
+	}
+	desc.Elem = desc.Type.Elem()
+	desc.Name = desc.Elem.PkgPath() + "." + desc.Elem.Name()
+	return
+}
+
+func Find[C Component](components []Component) C {
+	for _, c := range components {
+		if cc, ok := c.(C); ok {
+			return cc
+		}
+	}
+	panic("FindComponent: Invalid arguments")
+}
+
+// Put initializes a single component in the pool.
+//
+// Init may initialize components, but not call functions on them!
+func PutComponent[C Component](p *Pool, thread int32, core Core, init func(component C, thread int32)) C {
+	p.init()
+
+	p.rLock.Lock(int(thread))
+	defer p.rLock.Unlock()
+
+	// get a description of the type
+	cd := GetDescription[C]()
+
+	// find a field to put the component into
+	instance, created := func() (C, bool) {
+		p.cLock.Lock()
+		defer p.cLock.Unlock()
+
+		// create the component
+		field, ok := p.components[cd.Name]
+		if ok {
+			return field.(C), false
+		}
+
+		// create a new component
+		p.components[cd.Name] = cd.New().(Component)
+		return p.components[cd.Name].(C), true
+	}()
+
+	// if we already created the instance, then there is nothing to do
+	// as someone else will init it!
+	if !created {
+		return instance
+	}
+
+	// setup the core initialization now!
+	instance.getBase().Core = core
+
+	if init == nil {
+		return instance
+	}
+
+	// if we are in nested mode, then delay the init!
+	if !atomic.CompareAndSwapUint64(&p.nested, 0, 1) {
+		func() {
+			p.queue = append(p.queue, func(thread int32) {
+				init(instance, thread)
+			})
+		}()
+		return instance
+	}
+	defer atomic.StoreUint64(&p.nested, 0)
+
+	// init ourselves first (everything below will be nested)
+	init(instance, thread)
+
+	// do all the delayed initializations
+	index := 0
+	for len(p.queue) > index {
+		p.queue[index](thread)
+		index++
+	}
+	p.queue = nil
+
+	// and return the instance
+	return instance
+}

internal/component/snapshots/backup.go

@@ -0,0 +1,170 @@
+package snapshots
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/tkw1536/goprogram/status"
+	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
+)
+
+// Backup describes a backup
+type Backup struct {
+	Description BackupDescription
+
+	// Start and End Time of the backup
+	StartTime time.Time
+	EndTime   time.Time
+
+	// various error states, which are ignored when creating the snapshot
+	ErrPanic interface{}
+
+	// errors for the various components
+	ComponentErrors map[string]error
+
+	// TODO: Make this proper
+	ConfigFileErr error
+
+	// Snapshots containing instances
+	InstanceListErr   error
+	InstanceSnapshots []Snapshot
+
+	// List of files included
+	WithManifest
+}
+
+// BackupDescription provides a description for a backup
+type BackupDescription struct {
+	Dest string // Destination path
+	Auto bool   // Was the path created automatically?
+
+	ConcurrentSnapshots int // maximum number of concurrent snapshots
+}
+
+// New create a new Backup
+func (manager *Manager) NewBackup(io stream.IOStream, description BackupDescription) (backup Backup) {
+	backup.Description = description
+
+	// catch anything critical that happened during the snapshot
+	defer func() {
+		backup.ErrPanic = recover()
+	}()
+
+	// do the create keeping track of time!
+	logging.LogOperation(func() error {
+		backup.StartTime = time.Now().UTC()
+		backup.run(io, manager)
+		backup.EndTime = time.Now().UTC()
+
+		return nil
+	}, io, "Writing backup files")
+
+	return
+}
+
+func (backup *Backup) run(ios stream.IOStream, manager *Manager) {
+	// create a manifest
+	manifest, done := backup.handleManifest(backup.Description.Dest)
+	defer done()
+
+	// create a new status display
+	backups := manager.Backupable
+	backup.ComponentErrors = make(map[string]error, len(backups))
+
+	// Component backup tasks
+	logging.LogOperation(func() error {
+		st := status.NewWithCompat(ios.Stdout, 0)
+		st.Start()
+		defer st.Stop()
+
+		errors := status.Group[component.Backupable, error]{
+			PrefixString: func(item component.Backupable, index int) string {
+				return fmt.Sprintf("[backup %q]: ", item.Name())
+			},
+			PrefixAlign: true,
+
+			Handler: func(bc component.Backupable, index int, writer io.Writer) error {
+				return bc.Backup(
+					component.NewStagingContext(
+						manager.Environment,
+						stream.NewIOStream(writer, writer, nil, 0),
+						filepath.Join(backup.Description.Dest, bc.BackupName()),
+						manifest,
+					),
+				)
+			},
+
+			ResultString: status.DefaultErrorString[component.Backupable],
+		}.Use(st, backups)
+
+		for i, bc := range backups {
+			backup.ComponentErrors[bc.Name()] = errors[i]
+		}
+
+		return nil
+	}, ios, "Backing up core components")
+
+	// backup instances
+	logging.LogOperation(func() error {
+		st := status.NewWithCompat(ios.Stdout, 0)
+		st.Start()
+		defer st.Stop()
+
+		instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
+		if err := manager.Environment.Mkdir(instancesBackupDir, environment.DefaultDirPerm); err != nil {
+			backup.InstanceListErr = err
+			return nil
+		}
+
+		// list all instances
+		wissKIs, err := manager.Instances.All()
+		if err != nil {
+			backup.InstanceListErr = err
+			return nil
+		}
+
+		// make a backup of the snapshots
+		backup.InstanceSnapshots = status.Group[instances.WissKI, Snapshot]{
+			PrefixString: func(item instances.WissKI, index int) string {
+				return fmt.Sprintf("[snapshot %q]: ", item.Slug)
+			},
+			PrefixAlign: true,
+
+			Handler: func(instance instances.WissKI, index int, writer io.Writer) Snapshot {
+				dir := filepath.Join(instancesBackupDir, instance.Slug)
+				if err := manager.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil {
+					return Snapshot{
+						ErrPanic: err,
+					}
+				}
+
+				manifest <- dir
+
+				return manager.NewSnapshot(instance, stream.NewIOStream(writer, writer, nil, 0), SnapshotDescription{
+					Dest: dir,
+				})
+			},
+			ResultString: func(res Snapshot, item instances.WissKI, index int) string {
+				return "done"
+			},
+			WaitString:   status.DefaultWaitString[instances.WissKI],
+			HandlerLimit: backup.Description.ConcurrentSnapshots,
+		}.Use(st, wissKIs)
+
+		// sort the instances
+		slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
+			return a.Instance.Slug < b.Instance.Slug
+		})
+
+		return nil
+	}, ios, "Creating instance snapshots")
+
+}

internal/component/snapshots/manager.go

@@ -6,8 +6,7 @@ import (
 	"time"
 
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
-	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
-	"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/password"
@@ -16,9 +15,10 @@ import (
 // Manager manages snapshots and backups
 type Manager struct {
 	component.ComponentBase
+	Instances *instances.Instances
 
-	TS  *triplestore.Triplestore
-	SQL *sql.SQL
+	Snapshotable []component.Snapshotable
+	Backupable   []component.Backupable
 }
 
 func (Manager) Name() string { return "snapshots" }
@@ -68,6 +68,7 @@ func (*Manager) newSnapshotName(prefix string) string {
 func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
 	for path == "" || environment.IsExist(err) {
 		path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
+		fmt.Println("path =>", prefix, "err => ", err)
 		err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
 	}
 	if err != nil {

internal/component/snapshots/manifest.go

@@ -0,0 +1,33 @@
+package snapshots
+
+import (
+	"path/filepath"
+)
+
+type WithManifest struct {
+	Manifest []string
+}
+
+func (wm *WithManifest) handleManifest(dest string) (chan<- string, func()) {
+	manifest := make(chan string)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+
+		for file := range manifest {
+			// get the relative path to the root of the manifest.
+			// nothing *should* go wrong, but in case it does, use the original path.
+			path, err := filepath.Rel(dest, file)
+			if err != nil {
+				path = file
+			}
+
+			// add the manifest
+			wm.Manifest = append(wm.Manifest, path)
+		}
+	}()
+	return manifest, func() {
+		close(manifest)
+		<-done
+	}
+}

internal/component/snapshots/report.go

@@ -0,0 +1,154 @@
+package snapshots
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (snapshot Snapshot) String() string {
+	var builder strings.Builder
+	snapshot.Report(&builder)
+	return builder.String()
+}
+
+// Report writes a report from snapshot into w
+func (snapshot Snapshot) Report(w io.Writer) (int, error) {
+	ww := countwriter.NewCountWriter(w)
+
+	encoder := json.NewEncoder(ww)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
+
+	fmt.Fprintf(ww, "Slug:  %s\n", snapshot.Instance.Slug)
+	fmt.Fprintf(ww, "Dest:  %s\n", snapshot.Description.Dest)
+
+	fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
+	fmt.Fprintf(ww, "End:   %s\n", snapshot.EndTime)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Description =======\n")
+	encoder.Encode(snapshot.Description)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Instance =======\n")
+	encoder.Encode(snapshot.Instance)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Errors =======\n")
+	fmt.Fprintf(ww, "Panic:       %v\n", snapshot.ErrPanic)
+	fmt.Fprintf(ww, "Start:       %s\n", snapshot.ErrStart)
+	fmt.Fprintf(ww, "Stop:        %s\n", snapshot.ErrStop)
+
+	fmt.Fprintf(ww, "Whitebox:    %s\n", snapshot.ErrWhitebox)
+	fmt.Fprintf(ww, "Blackbox:    %s\n", snapshot.ErrBlackbox)
+
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Manifest =======\n")
+	for _, file := range snapshot.Manifest {
+		io.WriteString(ww, file+"\n")
+	}
+
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
+
+	return ww.Sum()
+}
+
+// WriteReport writes out the report belonging to this snapshot.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
+		stream.Println(reportPath)
+
+		// create the report file!
+		report, err := env.Create(reportPath, environment.DefaultFilePerm)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = io.WriteString(report, snapshot.String())
+		return err
+	}, stream, "Writing snapshot report")
+}
+
+// Strings turns this backup into a string for the BackupReport.
+func (backup Backup) String() string {
+	var builder strings.Builder
+	backup.Report(&builder)
+	return builder.String()
+}
+
+// Report formats a report for this backup, and writes it into Writer.
+func (backup Backup) Report(w io.Writer) (int, error) {
+	cw := countwriter.NewCountWriter(w)
+
+	encoder := json.NewEncoder(cw)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(cw, "======= Backup =======\n")
+
+	fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
+	fmt.Fprintf(cw, "End:   %s\n", backup.EndTime)
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Description =======\n")
+	encoder.Encode(backup.Description)
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Errors =======\n")
+	fmt.Fprintf(cw, "Panic:            %v\n", backup.ErrPanic)
+	fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
+	fmt.Fprintf(cw, "ConfigFileErr:    %s\n", backup.ConfigFileErr)
+	fmt.Fprintf(cw, "InstanceListErr:  %s\n", backup.InstanceListErr)
+
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Snapshots =======\n")
+	for _, s := range backup.InstanceSnapshots {
+		io.WriteString(cw, s.String())
+		io.WriteString(cw, "\n")
+	}
+
+	io.WriteString(cw, "======= Manifest =======\n")
+	for _, file := range backup.Manifest {
+		io.WriteString(cw, file+"\n")
+	}
+
+	io.WriteString(cw, "\n")
+
+	return cw.Sum()
+}
+
+// WriteReport writes out the report belonging to this backup.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(backup.Description.Dest, "report.txt")
+		stream.Println(reportPath)
+
+		// create the report file!
+		report, err := env.Create(reportPath, environment.DefaultFilePerm)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = io.WriteString(report, backup.String())
+		return err
+	}, stream, "Writing backup report")
+}

internal/component/snapshots/snapshot.go

@@ -0,0 +1,125 @@
+package snapshots
+
+import (
+	"fmt"
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
+	"github.com/tkw1536/goprogram/status"
+	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
+)
+
+// SnapshotDescription is a description for a snapshot
+type SnapshotDescription struct {
+	Dest      string // destination path
+	Log       bool   // should we log the creation of this snapshot?
+	Keepalive bool   // should we keep the instance alive while making the snapshot?
+}
+
+// Snapshot represents the result of generating a snapshot
+type Snapshot struct {
+	Description SnapshotDescription
+	Instance    models.Instance
+
+	// Start and End Time of the snapshot
+	StartTime time.Time
+	EndTime   time.Time
+
+	// Generic Panic that may have occured
+	ErrPanic    interface{}
+	ErrStart    error
+	ErrStop     error
+	ErrWhitebox map[string]error
+	ErrBlackbox map[string]error
+
+	// List of files included
+	WithManifest
+}
+
+// Snapshot creates a new snapshot of this instance into dest
+func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
+	// setup the snapshot
+	snapshot.Description = desc
+	snapshot.Instance = instance.Instance
+
+	// capture anything critical, and write the end time
+	defer func() {
+		snapshot.ErrPanic = recover()
+	}()
+
+	// do the create keeping track of time!
+	logging.LogOperation(func() error {
+		snapshot.StartTime = time.Now().UTC()
+
+		snapshot.ErrWhitebox = snapshot.makeParts(io, snapshots, instance, false)
+		snapshot.ErrBlackbox = snapshot.makeParts(io, snapshots, instance, true)
+
+		snapshot.EndTime = time.Now().UTC()
+		return nil
+	}, io, "Writing snapshot files")
+
+	slices.Sort(snapshot.Manifest)
+	return
+}
+
+func (snapshot *Snapshot) makeParts(ios stream.IOStream, snapshots *Manager, instance instances.WissKI, needsRunning bool) map[string]error {
+	if !needsRunning && !snapshot.Description.Keepalive {
+		stack := instance.Barrel()
+
+		logging.LogMessage(ios, "Stopping instance")
+		snapshot.ErrStop = stack.Down(ios)
+
+		defer func() {
+			logging.LogMessage(ios, "Starting instance")
+			snapshot.ErrStart = stack.Up(ios)
+		}()
+	}
+	// handle writing the manifest!
+	manifest, done := snapshot.handleManifest(snapshot.Description.Dest)
+	defer done()
+
+	// create a new status
+	st := status.NewWithCompat(ios.Stdout, 0)
+	st.Start()
+	defer st.Stop()
+
+	// get all the components
+	comps := slicesx.FilterClone(snapshots.Snapshotable, func(sc component.Snapshotable) bool {
+		return sc.SnapshotNeedsRunning() == needsRunning
+	})
+
+	results := make(map[string]error, len(comps))
+
+	errors := status.Group[component.Snapshotable, error]{
+		PrefixString: func(item component.Snapshotable, index int) string {
+			return fmt.Sprintf("[snapshot %q]: ", item.Name())
+		},
+		PrefixAlign: true,
+
+		Handler: func(sc component.Snapshotable, index int, writer io.Writer) error {
+			return sc.Snapshot(
+				instance.Instance,
+				component.NewStagingContext(
+					snapshots.Environment,
+					stream.NewIOStream(writer, writer, nil, 0),
+					filepath.Join(snapshot.Description.Dest, sc.SnapshotName()),
+					manifest,
+				),
+			)
+		},
+
+		ResultString: status.DefaultErrorString[component.Snapshotable],
+	}.Use(st, comps)
+
+	for i, wc := range comps {
+		results[wc.Name()] = errors[i]
+	}
+	return results
+}

internal/component/snapshots/snapshot_make.go

@@ -1,301 +0,0 @@
-package snapshots
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
-	"github.com/FAU-CDI/wisski-distillery/internal/models"
-	"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
-	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
-	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
-	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
-	"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
-	"github.com/tkw1536/goprogram/status"
-	"github.com/tkw1536/goprogram/stream"
-	"golang.org/x/exp/slices"
-)
-
-// SnapshotDescription is a description for a snapshot
-type SnapshotDescription struct {
-	Dest      string // destination path
-	Log       bool   // should we log the creation of this snapshot?
-	Keepalive bool   // should we keep the instance alive while making the snapshot?
-}
-
-// Snapshot represents the result of generating a snapshot
-type Snapshot struct {
-	Description SnapshotDescription
-	Instance    models.Instance
-
-	// Start and End Time of the snapshot
-	StartTime time.Time
-	EndTime   time.Time
-
-	// Generic Panic that may have occured
-	ErrPanic interface{}
-
-	// Errors during starting and stopping the system
-	ErrStart error
-	ErrStop  error
-
-	// List of files included
-	Manifest []string
-
-	// Errors during other parts
-	ErrBookkeep    error
-	ErrPathbuilder error
-	ErrFilesystem  error
-	ErrTriplestore error
-	ErrSQL         error
-}
-
-func (snapshot Snapshot) String() string {
-	var builder strings.Builder
-	snapshot.Report(&builder)
-	return builder.String()
-}
-
-// Report writes a report from snapshot into w
-func (snapshot Snapshot) Report(w io.Writer) (int, error) {
-	ww := countwriter.NewCountWriter(w)
-
-	// TODO: Errors of the writer!
-	encoder := json.NewEncoder(ww)
-	encoder.SetIndent("", "  ")
-
-	io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
-
-	fmt.Fprintf(ww, "Slug:  %s\n", snapshot.Instance.Slug)
-	fmt.Fprintf(ww, "Dest:  %s\n", snapshot.Description.Dest)
-
-	fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
-	fmt.Fprintf(ww, "End:   %s\n", snapshot.EndTime)
-	io.WriteString(ww, "\n")
-
-	io.WriteString(ww, "======= Description =======\n")
-	encoder.Encode(snapshot.Description)
-	io.WriteString(ww, "\n")
-
-	io.WriteString(ww, "======= Instance =======\n")
-	encoder.Encode(snapshot.Instance)
-	io.WriteString(ww, "\n")
-
-	io.WriteString(ww, "======= Errors =======\n")
-	fmt.Fprintf(ww, "Panic:       %v\n", snapshot.ErrPanic)
-	fmt.Fprintf(ww, "Start:       %s\n", snapshot.ErrStart)
-	fmt.Fprintf(ww, "Stop:        %s\n", snapshot.ErrStop)
-	fmt.Fprintf(ww, "Bookkeep:    %s\n", snapshot.ErrBookkeep)
-	fmt.Fprintf(ww, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
-	fmt.Fprintf(ww, "Filesystem:  %s\n", snapshot.ErrFilesystem)
-	fmt.Fprintf(ww, "Triplestore: %s\n", snapshot.ErrTriplestore)
-	fmt.Fprintf(ww, "SQL:         %s\n", snapshot.ErrSQL)
-	io.WriteString(ww, "\n")
-
-	io.WriteString(ww, "======= Manifest =======\n")
-	for _, file := range snapshot.Manifest {
-		io.WriteString(ww, file+"\n")
-	}
-
-	io.WriteString(ww, "\n")
-
-	io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
-
-	return ww.Sum()
-}
-
-// Snapshot creates a new snapshot of this instance into dest
-func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
-	// setup the snapshot
-	snapshot.Description = desc
-	snapshot.Instance = instance.Instance
-
-	// capture anything critical, and write the end time
-	defer func() {
-		snapshot.ErrPanic = recover()
-	}()
-
-	// do the create keeping track of time!
-	logging.LogOperation(func() error {
-		snapshot.StartTime = time.Now().UTC()
-
-		snapshot.makeBlackbox(io, snapshots, instance)
-		snapshot.makeWhitebox(io, snapshots, instance)
-
-		snapshot.EndTime = time.Now().UTC()
-		return nil
-	}, io, "Writing snapshot files")
-
-	slices.Sort(snapshot.Manifest)
-	return
-}
-
-// makeBlackbox runs the blackbox backup of the system.
-// It pauses the Instance, if a consistent state is required.
-func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
-	stack := instance.Barrel()
-
-	og := opgroup.NewOpGroup[string](4)
-
-	st := status.NewWithCompat(io.Stdout, 0)
-	st.Start()
-	defer st.Stop()
-
-	// stop the instance (unless it was explicitly asked to not do so!)
-	if !snapshot.Description.Keepalive {
-		logging.LogMessage(io, "Stopping instance")
-		snapshot.ErrStop = stack.Down(io)
-
-		defer func() {
-			logging.LogMessage(io, "Starting instance")
-			snapshot.ErrStart = stack.Up(io)
-		}()
-	}
-
-	// write bookkeeping information
-	og.GoErr(func(files chan<- string) error {
-		line := st.OpenLine("[snapshot bookkeeping]: ", "")
-		defer line.Close()
-		defer fmt.Fprintln(line, "done")
-
-		bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
-		fmt.Fprintln(line, bkPath)
-		files <- bkPath
-
-		info, err := snapshots.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
-		if err != nil {
-			return err
-		}
-		defer info.Close()
-
-		// print whatever is in the database
-		// TODO: This should be sql code, maybe gorm can do that?
-		_, err = fmt.Fprintf(info, "%#v\n", instance.Instance)
-		return err
-	}, &snapshot.ErrBookkeep)
-
-	// backup the filesystem
-	og.GoErr(func(files chan<- string) error {
-		line := st.OpenLine("[snapshot filesystem]: ", "")
-		defer line.Close()
-		defer fmt.Fprintln(line, "done")
-
-		fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
-
-		// copy over whatever is in the base directory
-		defer fmt.Fprintln(line, "done")
-		return fsx.CopyDirectory(snapshots.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
-			fmt.Fprintln(line, dst)
-			files <- dst
-		})
-
-	}, &snapshot.ErrFilesystem)
-
-	// backup the graph db repository
-	og.GoErr(func(files chan<- string) error {
-		line := st.OpenLine("[snapshot triplestore]: ", "")
-		defer line.Close()
-		defer fmt.Fprintln(line, "done")
-
-		tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
-		fmt.Fprintln(line, tsPath)
-		files <- tsPath
-
-		nquads, err := snapshots.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
-		if err != nil {
-			return err
-		}
-		defer nquads.Close()
-
-		// directly store the result
-		_, err = snapshots.TS.SnapshotDB(nquads, instance.GraphDBRepository)
-		return err
-	}, &snapshot.ErrTriplestore)
-
-	// backup the sql database
-	og.GoErr(func(files chan<- string) error {
-		line := st.OpenLine("[snapshot sql]: ", "")
-		defer line.Close()
-		defer fmt.Fprintln(line, "done")
-
-		sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
-		fmt.Fprintln(line, sqlPath)
-		files <- sqlPath
-
-		sql, err := snapshots.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
-		if err != nil {
-			return err
-		}
-		defer sql.Close()
-
-		// directly store the result
-		return snapshots.SQL.SnapshotDB(io, sql, instance.SqlDatabase)
-	}, &snapshot.ErrSQL)
-
-	// wait for the group!
-	snapshot.waitGroup(io, og)
-}
-
-// makeWhitebox runs the whitebox backup of the system.
-// The instance should be running during this step.
-func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
-	og := opgroup.NewOpGroup[string](1)
-
-	// write pathbuilders
-	og.GoErr(func(files chan<- string) error {
-
-		pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
-		files <- pbPath
-
-		// create the directory!
-		if err := snapshots.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
-			return err
-		}
-
-		// put in all the pathbuilders
-		return instance.ExportPathbuilders(pbPath)
-	}, &snapshot.ErrPathbuilder)
-
-	// wait for the group!
-	snapshot.waitGroup(io, og)
-}
-
-// waitGroup waits for the
-func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[string]) {
-	// wait for the messages to return
-	for file := range og.Wait() {
-		// get the relative path to the root of the manifest.
-		// nothing *should* go wrong, but in case it does, use the original path.
-		path, err := filepath.Rel(snapshot.Description.Dest, file)
-		if err != nil {
-			path = file
-		}
-
-		// add the manifest
-		snapshot.Manifest = append(snapshot.Manifest, path)
-	}
-}
-
-// WriteReport writes out the report belonging to this snapshot.
-// It is a separate function, to allow writing it indepenently of the rest.
-func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
-	return logging.LogOperation(func() error {
-		reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
-		stream.Println(reportPath)
-
-		// create the report file!
-		report, err := env.Create(reportPath, environment.DefaultFilePerm)
-		if err != nil {
-			return err
-		}
-		defer report.Close()
-
-		// print the report into it!
-		_, err = io.WriteString(report, snapshot.String())
-		return err
-	}, stream, "Writing snapshot report")
-}

internal/component/sql/snapshot.go

@@ -3,11 +3,25 @@ package sql
 import (
 	"io"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/tkw1536/goprogram/stream"
 )
 
+func (SQL) SnapshotNeedsRunning() bool { return false }
+
+func (SQL) SnapshotName() string { return "sql" }
+
+func (sql *SQL) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.AddDirectory(".", func() error {
+		return context.AddFile(wisski.SqlDatabase+".sql", func(file io.Writer) error {
+			return sql.SnapshotDB(context.IO(), file, wisski.SqlDatabase)
+		})
+	})
+}
+
 // SnapshotDB makes a backup of the sql database into dest.
-func (sql SQL) SnapshotDB(io stream.IOStream, dest io.Writer, database string) error {
+func (sql *SQL) SnapshotDB(io stream.IOStream, dest io.Writer, database string) error {
 	io = io.Streams(dest, nil, nil, 0).NonInteractive()
 
 	code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database)

internal/component/triplestore/database.go

@@ -122,21 +122,6 @@ func (ts Triplestore) PurgeRepo(repo string) error {
 	return nil
 }
 
-var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
-
-// SnapshotDB snapshots the provided repository into dst
-func (ts Triplestore) SnapshotDB(dst io.Writer, repo string) (int64, error) {
-	res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
-	if err != nil {
-		return 0, err
-	}
-	if res.StatusCode != http.StatusOK {
-		return 0, errTSBackupWrongStatusCode
-	}
-	defer res.Body.Close()
-	return io.Copy(dst, res.Body)
-}
-
 type Repository struct {
 	ID         string `json:"id"`
 	Title      string `json:"title"`

internal/component/triplestore/snapshot.go

@@ -0,0 +1,38 @@
+package triplestore
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/pkg/errors"
+)
+
+func (Triplestore) SnapshotNeedsRunning() bool { return false }
+
+func (Triplestore) SnapshotName() string { return "triplestore" }
+
+func (ts *Triplestore) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.AddDirectory(".", func() error {
+		return context.AddFile(wisski.GraphDBRepository+".nq", func(file io.Writer) error {
+			_, err := ts.SnapshotDB(file, wisski.GraphDBRepository)
+			return err
+		})
+	})
+}
+
+var errTSBackupWrongStatusCode = errors.New("Triplestore.Backup: Wrong status code")
+
+// SnapshotDB snapshots the provided repository into dst
+func (ts Triplestore) SnapshotDB(dst io.Writer, repo string) (int64, error) {
+	res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
+	if err != nil {
+		return 0, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return 0, errTSBackupWrongStatusCode
+	}
+	defer res.Body.Close()
+	return io.Copy(dst, res.Body)
+}