'wdcli backup': Rework backup process

This commit reworks the backup process to dynamically find the list of
components.

README.md

@@ -85,18 +85,19 @@ These are:
 
   - It is configured to run inside a docker container
   - A passwordless `root` account is created, which can only be used from inside the container.
+  - An additional admin account (as defined per config file) is created, which is used for administration.
   - A secondary management account is also created. This is configured via the distillery configuration file, and can be access from anywhere.
   - A `bookkeeping` database and table is created by default, to store known WissKI instance metadata in.
   - It is accsssible using `127.0.0.1:3306`
   - A database shell can be opened using `sudo /var/www/deploy/wdcli mysql`.
   - A [phpmyadmin](https://www.phpmyadmin.net/) is started on `127.0.0.1:8080`.
-  - See [distillery/resources/compose/sql](embed/resources/compose/sql) for implementation details.
+  - See [internal/component/sql](internal/component/sql) for implementation details.
 
 - [GraphDB](http://graphdb.ontotext.com/) - a SPARQL backend for WissKI (Version 10.0 or later)
   - It is configured to run inside a docker container.
   - The Workbench API is started on `127.0.0.1:7200`.
   - Security is not enabled at the moment.
-  - See [distillery/resources/compose/triplestore](embed/resources/compose/triplestore) for implementation details.
+  - See [internal/component/triplestore](internal/component/triplestore) for implementation details.
 
 - [proxyssh](https://github.com/tkw1536/proxyssh) - an ssh server that delegates client connections to different WissKIs
   - It is configured to run inside a docker container.

cmd/system_update.go

@@ -122,7 +122,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
 	}
 
 	if err := logging.LogOperation(func() error {
-		for _, component := range dis.Components() {
+		for _, component := range dis.Installables() {
 			stack := component.Stack()
 			ctx := component.Context(ctx)
 			if err := logging.LogOperation(func() error {

internal/component/component.go

@@ -3,6 +3,7 @@ package component
 
 import (
 	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/tkw1536/goprogram/stream"
 )
 
 // Component represents a logical subsystem of the distillery.
@@ -31,20 +32,34 @@ type Component interface {
 	Base() *ComponentBase
 }
 
-// InstallableComponent implements an installable component
+// Installable implements an installable component.
-type InstallableComponent interface {
+type Installable interface {
 	Component
 
 	// Stack can be used to gain access to the "docker compose" stack.
 	//
-	// This should internally call
+	// This should internally call [ComponentBase.MakeStack]
-	Stack() Installable
+	Stack() StackWithResources
 
 	// Context returns a new InstallationContext to be used during installation from the command line.
 	// Typically this should just pass through the parent, but might perform other tasks.
 	Context(parent InstallationContext) InstallationContext
 }
 
+// Backupable represents a component with a Backup method
+type Backupable interface {
+	Component
+
+	// BackupName returns a new name to be used as an argument for path.
+	BackupName() string
+
+	// Backup backs up this component into the destination path path.
+	//
+	// The destination path may be a folder or directory, depending on the component.
+	// The destination path does not need to exist.
+	Backup(io stream.IOStream, path string) error
+}
+
 // ComponentBase implements base functionality for a component
 type ComponentBase struct {
 	Dir    string         // Dir is the directory this component lives in
@@ -67,7 +82,7 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext {
 }
 
 // MakeStack registers the Installable as a stack
-func (cb ComponentBase) MakeStack(stack Installable) Installable {
+func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources {
 	stack.Dir = cb.Dir
 	return stack
 }

internal/component/control/backup.go

@@ -0,0 +1,48 @@
+package control
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (*Control) BackupName() string {
+	return "config"
+}
+
+// Backup backups all control plane configuration files into dest
+func (control *Control) Backup(io stream.IOStream, dest string) error {
+	// create the destination directory, TODO: outsource this
+	if err := os.Mkdir(dest, fs.ModeDir); err != nil {
+		return err
+	}
+
+	files := control.backupFiles()
+	for _, src := range files {
+		dst := filepath.Join(dest, filepath.Base(src)) // destination path
+
+		// if the src file does not exist, don't copy it!
+		if !fsx.IsFile(src) { // TODO: log this somewhere
+			continue
+		}
+
+		if err := fsx.CopyFile(dst, src); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// backupfiles lists the files to be backed up.
+func (control *Control) backupFiles() []string {
+	return []string{
+		control.Config.ConfigPath,
+		control.Config.ExecutablePath(),
+		control.Config.SelfOverridesFile,
+		control.Config.GlobalAuthorizedKeysFile,
+	}
+}

internal/component/control/control.go

@@ -0,0 +1,54 @@
+package control
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
+)
+
+// Control represents the control server
+type Control struct {
+	component.ComponentBase
+
+	Instances *instances.Instances
+
+	ResolverFile string
+}
+
+func (control Control) Name() string {
+	return "dis" // TODO: Rename this to control!
+}
+
+//go:embed all:control control.env
+var resources embed.FS
+
+func (control Control) Stack() component.StackWithResources {
+	return control.ComponentBase.MakeStack(component.StackWithResources{
+		Resources:   resources,
+		ContextPath: "control",
+		EnvPath:     "control.env",
+
+		EnvContext: map[string]string{
+			"VIRTUAL_HOST":      control.Config.DefaultHost(),
+			"LETSENCRYPT_HOST":  control.Config.DefaultSSLHost(),
+			"LETSENCRYPT_EMAIL": control.Config.CertbotEmail,
+
+			"CONFIG_PATH": control.Config.ConfigPath,
+			"DEPLOY_ROOT": control.Config.DeployRoot,
+
+			"GLOBAL_AUTHORIZED_KEYS_FILE": control.Config.GlobalAuthorizedKeysFile,
+			"SELF_OVERRIDES_FILE":         control.Config.SelfOverridesFile,
+		},
+
+		TouchFiles:       []string{control.ResolverFile},
+		CopyContextFiles: []string{core.Executable},
+	})
+}
+
+func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
+	return component.InstallationContext{
+		core.Executable: control.Config.CurrentExecutable(),
+	}
+}

internal/component/control/info.go

@@ -1,4 +1,4 @@
-package dis
+package control
 
 import (
 	"embed"
@@ -14,11 +14,9 @@ import (
 	"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
 	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/sync/errgroup"
-
-	_ "embed"
 )
 
-func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
+func (control *Control) info(io stream.IOStream) (http.Handler, error) {
 	mux := http.NewServeMux()
 
 	// handle everything under /dis/!
@@ -31,7 +29,7 @@ func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
 	})
 
 	// static stuff
-	static, err := dis.disStatic()
+	static, err := control.disStatic()
 	if err != nil {
 		return nil, err
 	}
@@ -39,22 +37,22 @@ func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
 
 	// render everything
 	mux.Handle("/dis/index", httpx.HTMLHandler[disIndex]{
-		Handler:  dis.disIndex,
+		Handler:  control.disIndex,
 		Template: indexTemplate,
 	})
 
 	mux.Handle("/dis/instance/", httpx.HTMLHandler[disInstance]{
-		Handler:  dis.disInstance,
+		Handler:  control.disInstance,
 		Template: instanceTemplate,
 	})
 
 	// api -- for future usage
-	mux.Handle("/dis/api/v1/instance/get/", httpx.JSON(dis.getinstance))
+	mux.Handle("/dis/api/v1/instance/get/", httpx.JSON(control.getinstance))
-	mux.Handle("/dis/api/v1/instance/all", httpx.JSON(dis.allinstances))
+	mux.Handle("/dis/api/v1/instance/all", httpx.JSON(control.allinstances))
 
 	// ensure that everyone is logged in!
 	return httpx.BasicAuth(mux, "WissKI Distillery Admin", func(user, pass string) bool {
-		return user == dis.Config.DisAdminUser && pass == dis.Config.DisAdminPassword
+		return user == control.Config.DisAdminUser && pass == control.Config.DisAdminPassword
 	}), nil
 }
 
@@ -70,7 +68,7 @@ type disIndex struct {
 	StoppedCount int
 }
 
-func (dis *Dis) disIndex(r *http.Request) (idx disIndex, err error) {
+func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
 	// load instances
 	idx.Instances, err = dis.allinstances(r)
 	if err != nil {
@@ -104,7 +102,7 @@ type disInstance struct {
 	Info     instances.Info
 }
 
-func (dis *Dis) disInstance(r *http.Request) (is disInstance, err error) {
+func (dis *Control) disInstance(r *http.Request) (is disInstance, err error) {
 	// find the slug as the last component of path!
 	slug := strings.TrimSuffix(r.URL.Path, "/")
 	slug = slug[strings.LastIndex(slug, "/")+1:]
@@ -134,7 +132,7 @@ func (dis *Dis) disInstance(r *http.Request) (is disInstance, err error) {
 //go:embed html/static
 var htmlStaticFS embed.FS
 
-func (*Dis) disStatic() (http.Handler, error) {
+func (*Control) disStatic() (http.Handler, error) {
 	fs, err := fs.Sub(htmlStaticFS, "html/static")
 	if err != nil {
 		return nil, err
@@ -151,7 +149,7 @@ var indexTemplate = template.Must(template.New("index.html").Parse(indexTemplate
 var instanceTemplateString string
 var instanceTemplate = template.Must(template.New("instance.html").Parse(instanceTemplateString))
 
-func (dis *Dis) getinstance(r *http.Request) (info instances.Info, err error) {
+func (dis *Control) getinstance(r *http.Request) (info instances.Info, err error) {
 	// find the slug as the last component of path!
 	slug := strings.TrimSuffix(r.URL.Path, "/")
 	slug = slug[strings.LastIndex(slug, "/")+1:]
@@ -169,7 +167,7 @@ func (dis *Dis) getinstance(r *http.Request) (info instances.Info, err error) {
 	return wisski.Info(false)
 }
 
-func (dis *Dis) allinstances(*http.Request) (infos []instances.Info, err error) {
+func (dis *Control) allinstances(*http.Request) (infos []instances.Info, err error) {
 	var errgroup errgroup.Group
 
 	// list all the instances

internal/component/control/resolver.go

@@ -1,4 +1,4 @@
-package dis
+package control
 
 import (
 	"fmt"
@@ -11,11 +11,11 @@ import (
 	"github.com/tkw1536/goprogram/stream"
 )
 
-func (dis Dis) ResolverConfigPath() string {
+func (control Control) ResolverConfigPath() string {
-	return filepath.Join(dis.Dir, dis.ResolverFile)
+	return filepath.Join(control.Dir, control.ResolverFile)
 }
 
-func (dis Dis) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
+func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
 	p.TrustXForwardedProto = true
 
 	fallback := &resolvers.Regexp{
@@ -23,20 +23,20 @@ func (dis Dis) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err err
 	}
 
 	// handle the default domain name!
-	domainName := dis.Config.DefaultDomain
+	domainName := control.Config.DefaultDomain
 	if domainName != "" {
 		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
 		io.Printf("registering default domain %s\n", domainName)
 	}
 
 	// handle the extra domains!
-	for _, domain := range dis.Config.SelfExtraDomains {
+	for _, domain := range control.Config.SelfExtraDomains {
 		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
 		io.Printf("registering legacy domain %s\n", domain)
 	}
 
 	// open the prefix file
-	prefixFile := dis.ResolverConfigPath()
+	prefixFile := control.ResolverConfigPath()
 	fs, err := os.Open(prefixFile)
 	io.Println("loading prefixes from ", prefixFile)
 	if err != nil {

internal/component/control/self.go

@@ -1,4 +1,4 @@
-package dis
+package control
 
 import (
 	"encoding/json"
@@ -11,10 +11,10 @@ import (
 )
 
 // self returns the handler for the self overrides
-func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
+func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
 	// open the overrides file
-	overrides, err := os.Open(dis.Config.SelfOverridesFile)
+	overrides, err := os.Open(control.Config.SelfOverridesFile)
-	io.Printf("loading overrides from %q\n", dis.Config.SelfOverridesFile)
+	io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
 	if err != nil {
 		return redirect, err
 	}
@@ -28,10 +28,10 @@ func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
 	if redirect.Overrides == nil {
 		redirect.Overrides = make(map[string]string)
 	}
-	redirect.Overrides[""] = dis.Config.SelfRedirect.String()
+	redirect.Overrides[""] = control.Config.SelfRedirect.String()
 
 	// create a redirect server
-	redirect.Fallback, err = dis.selfFallback()
+	redirect.Fallback, err = control.selfFallback()
 	if err != nil {
 		return redirect, err
 	}
@@ -42,24 +42,22 @@ func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
 	return redirect, nil
 }
 
-func (dis *Dis) selfFallback() (http.Handler, error) {
+func (control *Control) selfFallback() (http.Handler, error) {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	return http.HandlerFunc(control.serveFallback), nil
-		dis.serveFallback(w, r)
-	}), nil
 }
 
 var notFoundText = []byte("not found")
 
-func (dis *Dis) serveFallback(w http.ResponseWriter, r *http.Request) {
+func (control *Control) serveFallback(w http.ResponseWriter, r *http.Request) {
 
-	slug := dis.Config.SlugFromHost(r.Host)
+	slug := control.Config.SlugFromHost(r.Host)
 	if slug == "" {
 		w.WriteHeader(http.StatusNotFound)
 		w.Write(notFoundText)
 		return
 	}
 
-	if ok, _ := dis.Instances.Has(slug); !ok {
+	if ok, _ := control.Instances.Has(slug); !ok {
 		w.WriteHeader(http.StatusNotFound)
 		fmt.Fprintf(w, "WissKI %q not found\n", slug)
 		return

internal/component/control/server.go

@@ -1,4 +1,4 @@
-package dis
+package control
 
 import (
 	"net/http"
@@ -7,19 +7,19 @@ import (
 )
 
 // Server returns an http.Mux that implements the main server instance
-func (dis Dis) Server(io stream.IOStream) (http.Handler, error) {
+func (control Control) Server(io stream.IOStream) (http.Handler, error) {
 	// self server
-	self, err := dis.self(io)
+	self, err := control.self(io)
 	if err != nil {
 		return nil, err
 	}
 
-	resolver, err := dis.resolver(io)
+	resolver, err := control.resolver(io)
 	if err != nil {
 		return nil, err
 	}
 
-	info, err := dis.info(io)
+	info, err := control.info(io)
 	if err != nil {
 		return nil, err
 	}

internal/component/dis/dis.go

@@ -1,53 +0,0 @@
-package dis
-
-import (
-	"embed"
-
-	"github.com/FAU-CDI/wisski-distillery/internal/component"
-	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
-	"github.com/FAU-CDI/wisski-distillery/internal/core"
-)
-
-type Dis struct {
-	component.ComponentBase
-
-	Instances *instances.Instances
-
-	ResolverFile string
-}
-
-func (dis Dis) Name() string {
-	return "dis"
-}
-
-//go:embed all:stack dis.env
-var resources embed.FS
-
-func (dis Dis) Stack() component.Installable {
-	return dis.ComponentBase.MakeStack(component.Installable{
-		Resources:   resources,
-		ContextPath: "stack",
-		EnvPath:     "dis.env",
-
-		EnvContext: map[string]string{
-			"VIRTUAL_HOST":      dis.Config.DefaultHost(),
-			"LETSENCRYPT_HOST":  dis.Config.DefaultSSLHost(),
-			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
-
-			"CONFIG_PATH": dis.Config.ConfigPath,
-			"DEPLOY_ROOT": dis.Config.DeployRoot,
-
-			"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
-			"SELF_OVERRIDES_FILE":         dis.Config.SelfOverridesFile,
-		},
-
-		TouchFiles:       []string{dis.ResolverFile},
-		CopyContextFiles: []string{core.Executable},
-	})
-}
-
-func (dis Dis) Context(parent component.InstallationContext) component.InstallationContext {
-	return component.InstallationContext{
-		core.Executable: dis.Config.CurrentExecutable(),
-	}
-}

internal/component/installable.go

@@ -13,9 +13,9 @@ import (
 
 // TODO: Move this package into components
 
-// Installable represents a Stack that can be automatically installed from a set of resources
+// StackWithResources represents a Stack that can be automatically installed from a set of resources.
 // See the [Install] method.
-type Installable struct {
+type StackWithResources struct {
 	Stack
 
 	// Installable enabled installing several resources from a (potentially embedded) filesystem.
@@ -42,7 +42,7 @@ type InstallationContext map[string]string
 //
 // Installation is non-interactive, but will provide debugging output onto io.
 // InstallationContext
-func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
+func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
 	if is.ContextPath != "" {
 		// setup the base files
 		if err := unpack.InstallDir(

internal/component/instances/wisski_stack.go

@@ -12,8 +12,8 @@ import (
 var barrelResources embed.FS
 
 // Barrel returns a stack representing the running WissKI Instance
-func (wisski WissKI) Barrel() component.Installable {
+func (wisski WissKI) Barrel() component.StackWithResources {
-	return component.Installable{
+	return component.StackWithResources{
 		Stack: component.Stack{
 			Dir: wisski.FilesystemBase,
 		},
@@ -48,8 +48,8 @@ func (wisski WissKI) Barrel() component.Installable {
 var reserveResources embed.FS
 
 // Reserve returns a stack representing the reserve instance
-func (wisski WissKI) Reserve() component.Installable {
+func (wisski WissKI) Reserve() component.StackWithResources {
-	return component.Installable{
+	return component.StackWithResources{
 		Stack: component.Stack{
 			Dir: wisski.FilesystemBase,
 		},

internal/component/sql/backup.go

@@ -0,0 +1,35 @@
+package sql
+
+import (
+	"errors"
+	"os"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
+
+func (*SQL) BackupName() string {
+	return "sql.sql"
+}
+
+// Backup makes a backup of all SQL databases into the path dest.
+func (sql *SQL) Backup(io stream.IOStream, dest string) error {
+	// open the file, TODO: Outsource this to context
+	writer, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+
+	// run sqldump
+	io = io.Streams(writer, nil, nil, 0).NonInteractive()
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}

internal/component/sql/database.go

@@ -59,10 +59,8 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
 	return table, nil
 }
 
-var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
+// Snapshot makes a backup of the sql database into dest.
-
+func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error {
-// Backup makes a backup of the sql database into dest.
-func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
 	io = io.Streams(dest, nil, nil, 0).NonInteractive()
 
 	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
@@ -75,20 +73,6 @@ func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error
 	return nil
 }
 
-// BackupAll makes a backup of all sql databases
-func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
-	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
-
-	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
-	if err != nil {
-		return err
-	}
-	if code != 0 {
-		return errSQLBackup
-	}
-	return nil
-}
-
 // OpenShell executes a mysql shell command
 func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
 	return sql.Stack().Exec(io, "sql", "mysql", argv...)

internal/component/sql/sql.go

@@ -22,13 +22,13 @@ func (SQL) Name() string {
 	return "sql"
 }
 
-//go:embed all:stack
+//go:embed all:sql
 var resources embed.FS
 
-func (ssh SQL) Stack() component.Installable {
+func (ssh SQL) Stack() component.StackWithResources {
-	return ssh.ComponentBase.MakeStack(component.Installable{
+	return ssh.ComponentBase.MakeStack(component.StackWithResources{
 		Resources:   resources,
-		ContextPath: "stack",
+		ContextPath: "sql",
 
 		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
 		MakeDirs: []string{

internal/component/ssh/ssh.go

@@ -17,8 +17,8 @@ func (SSH) Name() string {
 //go:embed all:stack
 var resources embed.FS
 
-func (ssh SSH) Stack() component.Installable {
+func (ssh SSH) Stack() component.StackWithResources {
-	return ssh.ComponentBase.MakeStack(component.Installable{
+	return ssh.ComponentBase.MakeStack(component.StackWithResources{
 		Resources:   resources,
 		ContextPath: "stack",
 	})

internal/component/triplestore/backup.go

@@ -0,0 +1,58 @@
+package triplestore
+
+import (
+	"encoding/json"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (ts *Triplestore) BackupName() string { return "triplestore" }
+
+// Backup makes a backup of all Triplestore repositories databases into the path dest.
+func (ts *Triplestore) Backup(io stream.IOStream, dest string) error {
+
+	// list all the repositories
+	repos, err := ts.listRepositories()
+	if err != nil {
+		return err
+	}
+
+	// create the base directory, todo: outsource this
+	if err := os.Mkdir(dest, fs.ModeDir); err != nil {
+		return err
+	}
+
+	// iterate over all the repositories
+	for _, repo := range repos {
+		if rErr := (func(repo Repository) error {
+			name := filepath.Join(dest, repo.ID+".nq")
+
+			// todo: outsource this
+			dest, err := os.Create(name)
+			if err != nil {
+				return err
+			}
+			defer dest.Close()
+
+			_, err = ts.Snapshot(dest, repo.ID)
+			return err
+		}(repo)); err == nil && rErr != nil {
+			err = rErr
+		}
+	}
+	return err
+}
+
+func (ts Triplestore) listRepositories() (repos []Repository, err error) {
+	res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	err = json.NewDecoder(res.Body).Decode(&repos)
+	return
+}

internal/component/triplestore/database.go

@@ -5,11 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/fs"
 	"mime/multipart"
 	"net/http"
-	"os"
-	"path/filepath"
 
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/wait"
@@ -121,8 +118,8 @@ func (ts Triplestore) PurgeRepo(repo string) error {
 
 var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
 
-// TriplestoreBackup backs up the repository named repo into the writer dst.
+// Snapshot snapshots the provided repository into dst
-func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
+func (ts Triplestore) Snapshot(dst io.Writer, repo string) (int64, error) {
 	res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
 	if err != nil {
 		return 0, err
@@ -146,50 +143,6 @@ type Repository struct {
 	Local      bool   `json:"local"`
 }
 
-func (ts Triplestore) listRepositories() (repos []Repository, err error) {
-	res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
-	if err != nil {
-		return nil, err
-	}
-	defer res.Body.Close()
-
-	err = json.NewDecoder(res.Body).Decode(&repos)
-	return
-}
-
-// TriplestoreBackup backs up every graphdb instance into dst
-func (ts Triplestore) BackupAll(dst string) error {
-	// list all the repositories
-	repos, err := ts.listRepositories()
-	if err != nil {
-		return err
-	}
-
-	// create the base directory
-	if err := os.Mkdir(dst, fs.ModeDir); err != nil {
-		return err
-	}
-
-	// iterate over all the repositories
-	for _, repo := range repos {
-		if rErr := (func(repo Repository) error {
-			name := filepath.Join(dst, repo.ID+".nq")
-
-			dest, err := os.Create(name)
-			if err != nil {
-				return err
-			}
-			defer dest.Close()
-
-			_, err = ts.Backup(dest, repo.ID)
-			return err
-		}(repo)); err == nil && rErr != nil {
-			err = rErr
-		}
-	}
-	return err
-}
-
 var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
 
 func (ts Triplestore) Bootstrap(io stream.IOStream) error {

internal/component/triplestore/triplestore.go

@@ -26,8 +26,8 @@ func (Triplestore) Name() string {
 //go:embed all:stack
 var resources embed.FS
 
-func (ts Triplestore) Stack() component.Installable {
+func (ts Triplestore) Stack() component.StackWithResources {
-	return ts.ComponentBase.MakeStack(component.Installable{
+	return ts.ComponentBase.MakeStack(component.StackWithResources{
 		Resources:   resources,
 		ContextPath: "stack",
 

internal/component/web/web.go

@@ -17,7 +17,7 @@ func (Web) Name() string {
 	return "web"
 }
 
-func (web Web) Stack() component.Installable {
+func (web Web) Stack() component.StackWithResources {
 	if web.Config.HTTPSEnabled() {
 		return web.stackHTTPS()
 	} else {
@@ -29,8 +29,8 @@ func (web Web) Stack() component.Installable {
 //go:embed web-https.env
 var httpsResources embed.FS
 
-func (web Web) stackHTTPS() component.Installable {
+func (web Web) stackHTTPS() component.StackWithResources {
-	return web.MakeStack(component.Installable{
+	return web.MakeStack(component.StackWithResources{
 		Resources:   httpsResources,
 		ContextPath: "web-https",
 		EnvPath:     "web-https.env",
@@ -45,8 +45,8 @@ func (web Web) stackHTTPS() component.Installable {
 //go:embed web-http.env
 var httpResources embed.FS
 
-func (web Web) stackHTTP() component.Installable {
+func (web Web) stackHTTP() component.StackWithResources {
-	return web.MakeStack(component.Installable{
+	return web.MakeStack(component.StackWithResources{
 		Resources:   httpResources,
 		ContextPath: "web-http",
 		EnvPath:     "web-http.env",

internal/wisski/backup.go

@@ -11,11 +11,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
 	"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
-	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
-	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/exp/slices"
 )
@@ -36,13 +34,14 @@ type Backup struct {
 	// various error states, which are ignored when creating the snapshot
 	ErrPanic interface{}
 
+	// errors for the various components
+	ComponentErrors map[string]error
+
 	// SQL and triplestore errors
-	SQLErr error
+	TSErr error
-	TSErr  error
 
 	// TODO: Make this proper
-	ConfigFileErr       error
+	ConfigFileErr error
-	ConfigFilesManifest map[string]error
 
 	// Snapshots containing instances
 	InstanceListErr   error
@@ -76,17 +75,13 @@ func (backup Backup) Report(w io.Writer) (int, error) {
 	io.WriteString(cw, "\n")
 
 	io.WriteString(cw, "======= Errors =======\n")
-	fmt.Fprintf(cw, "Panic:           %v\n", backup.ErrPanic)
+	fmt.Fprintf(cw, "Panic:            %v\n", backup.ErrPanic)
-	fmt.Fprintf(cw, "SQLErr:          %s\n", backup.SQLErr)
+	fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
-	fmt.Fprintf(cw, "TSErr:           %s\n", backup.TSErr)
+	fmt.Fprintf(cw, "ConfigFileErr:    %s\n", backup.ConfigFileErr)
-	fmt.Fprintf(cw, "ConfigFileErr:   %s\n", backup.ConfigFileErr)
+	fmt.Fprintf(cw, "InstanceListErr:  %s\n", backup.InstanceListErr)
-	fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
 
 	io.WriteString(cw, "\n")
 
-	io.WriteString(cw, "======= Config Files =======\n")
-	encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
-
 	io.WriteString(cw, "======= Snapshots =======\n")
 	for _, s := range backup.InstanceSnapshots {
 		io.WriteString(cw, s.String())
@@ -103,6 +98,8 @@ func (backup Backup) Report(w io.Writer) (int, error) {
 	return cw.Sum()
 }
 
+// Backup makes a makes of the entire distillery.
+// To make a backup, all [BackupComponents] will be invoked.
 func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
 	backup.Description = description
 
@@ -123,75 +120,36 @@ func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription)
 	return
 }
 
-var errBackupSkipFile = errors.New("<file not found>")
+type backupResult struct {
+	name string
+	err  error
+}
 
 func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
-	// create a wait group, and message channel
-	wg := &sync.WaitGroup{}
-	files := make(chan string, 4)
 
-	// backup the sql
+	backups := dis.Backupable()
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
 
-		sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
+	files := make(chan string, len(backups))         // channel for files being added into the backups
-		files <- sqlPath
+	results := make(chan backupResult, len(backups)) // channel for results to be stored into
+	backup.ComponentErrors = make(map[string]error, len(backups))
 
-		sql, err := os.Create(sqlPath)
+	wg := &sync.WaitGroup{} // to wait for the results
-		if err != nil {
+	wg.Add(len(backups))
-			backup.SQLErr = err
+	for _, bc := range backups {
-			return
+		go func(bc component.Backupable) {
-		}
+			defer wg.Done()
-		defer sql.Close()
 
-		// directly store the result
+			// find the backup destination
-		backup.SQLErr = dis.SQL().BackupAll(io, sql)
+			dest := filepath.Join(backup.Description.Dest, bc.BackupName())
-	}()
+			files <- dest
 
-	// backup the triplestore
+			// make the backup and send the result!
-	wg.Add(1)
+			results <- backupResult{
-	go func() {
+				name: bc.Name(),
-		defer wg.Done()
+				err:  bc.Backup(io, dest),
-
-		tsPath := filepath.Join(backup.Description.Dest, "triplestore")
-		files <- tsPath
-
-		// directly store the result
-		backup.TSErr = dis.Triplestore().BackupAll(tsPath)
-	}()
-
-	// backup configuration files
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-
-		cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
-		if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
-			backup.ConfigFileErr = err
-			return
-		}
-
-		configs := []string{
-			dis.Config.ConfigPath,
-			filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
-			dis.Config.SelfOverridesFile,
-			dis.Config.GlobalAuthorizedKeysFile,
-		}
-
-		backup.ConfigFilesManifest = make(map[string]error, len(configs))
-		for _, src := range configs {
-			if !fsx.IsFile(src) {
-				backup.ConfigFilesManifest[src] = errBackupSkipFile
-				continue
 			}
-			dest := filepath.Join(cfgBackupDir, filepath.Base(src))
+		}(bc)
-
+	}
-			// copy the config file and store the error message
-			files <- src
-			backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
-		}
-	}()
 
 	// backup instances
 	wg.Add(1)
@@ -230,10 +188,18 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 
 	}()
 
-	// wait for the group, then close the message channel.
+	// finish processing all the results as soon as the group is done.
 	go func() {
+		defer close(results)
 		wg.Wait()
-		close(files)
+	}()
+
+	// finish the message processing once results are finished.
+	go func() {
+		defer close(files) // no more file processing!
+		for result := range results {
+			backup.ComponentErrors[result.name] = result.err
+		}
 	}()
 
 	for file := range files {

internal/wisski/component.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
-	"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/control"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
@@ -23,11 +23,11 @@ import (
 type components struct {
 
 	// installable components
-	web lazy.Lazy[*web.Web]
+	web     lazy.Lazy[*web.Web]
-	dis lazy.Lazy[*dis.Dis]
+	control lazy.Lazy[*control.Control]
-	ssh lazy.Lazy[*ssh.SSH]
+	ssh     lazy.Lazy[*ssh.SSH]
-	ts  lazy.Lazy[*triplestore.Triplestore]
+	ts      lazy.Lazy[*triplestore.Triplestore]
-	sql lazy.Lazy[*sql.SQL]
+	sql     lazy.Lazy[*sql.SQL]
 
 	// other components
 	instances lazy.Lazy[*instances.Instances]
@@ -67,23 +67,48 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
 	})
 }
 
-// Components returns all components that have a stack function
+func (dis *Distillery) ComponentsX() []component.Component {
-func (dis *Distillery) Components() []component.InstallableComponent {
+	return []component.Component{
-	return []component.InstallableComponent{
 		dis.Web(),
 		dis.Dis(),
 		dis.SSH(),
 		dis.Triplestore(),
 		dis.SQL(),
+		dis.Instances(),
 	}
 }
 
+// Backupable returns all the components that can be backuped up.
+func (dis *Distillery) Backupable() []component.Backupable {
+	return getComponents[component.Backupable](dis)
+}
+
+// Installables returns all components that can be installed
+func (dis *Distillery) Installables() []component.Installable {
+	return getComponents[component.Installable](dis)
+}
+
+func getComponents[C component.Component](dis *Distillery) (result []C) {
+	all := dis.ComponentsX()
+
+	result = make([]C, 0, len(all))
+	for _, c := range all {
+		sc, ok := c.(C)
+		if !ok {
+			continue
+		}
+		result = append(result, sc)
+	}
+
+	return
+}
+
 func (dis *Distillery) Web() *web.Web {
 	return makeComponent(dis, &dis.components.web, nil)
 }
 
-func (d *Distillery) Dis() *dis.Dis {
+func (d *Distillery) Dis() *control.Control {
-	return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
+	return makeComponent(d, &d.components.control, func(ddis *control.Control) {
 		ddis.ResolverFile = core.PrefixConfig
 		ddis.Instances = d.Instances()
 	})

internal/wisski/snapshot.go

@@ -244,7 +244,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
 		defer nquads.Close()
 
 		// directly store the result
-		_, err = dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
+		_, err = dis.Triplestore().Snapshot(nquads, instance.GraphDBRepository)
 		return err
 	}, &snapshot.ErrTriplestore)
 
@@ -260,7 +260,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
 		defer sql.Close()
 
 		// directly store the result
-		return dis.SQL().Backup(io, sql, instance.SqlDatabase)
+		return dis.SQL().Snapshot(io, sql, instance.SqlDatabase)
 	}, &snapshot.ErrSQL)
 
 	// wait for the group!