wdcli: Implement backup & snapshot

This commit implements command backup and snapshot.

TODO.md

@@ -28,28 +28,12 @@ Work in progress.
 - restructure resource files
 - Documentation
 - single malt
+- snapshots: export xml from pathbuilder
 
 ## Migrating Individual Commands
-- [ ] backup_all.sh
+
-- [x] backup_instance.sh
-- [x] blind_update.sh
-- [x] blind_update_all.sh
-- [x] cron_all.sh
-- [x] info.sh
-- [x] ls.sh
-- [x] make_mysql_account.sh
 - [ ] monday_full.sh
 - [ ] monday_short.sh
-- [x] mysql.sh
-- [x] provision.sh
-- [x] purge.sh
-- [x] rebuild.sh
-- [x] rebuild_all.sh
-- [x] reserve.sh
-- [x] shell.sh
-- [x] system_install.sh
-- [x] system_update.sh
-- [x] update_prefix_config.sh
 
 ## TO BE REMOVED
 - [ ] call_update_php_hack.sh

cmd/backup.go

@@ -0,0 +1,116 @@
+package cmd
+
+import (
+	"io/fs"
+	"os"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/targz"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Backup is the 'backup' command
+var Backup wisski_distillery.Command = backup{}
+
+type backup struct {
+	StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"`
+	Positionals struct {
+		Dest string `positional-arg-name:"DEST" description:"Destination path to write backup archive to. Defaults to the snapshots/archives/ directory"`
+	} `positional-args:"true"`
+}
+
+func (backup) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "backup",
+		Description: "Makes a backup of the entire distillery",
+	}
+}
+
+var errBackupFailed = exit.Error{
+	Message:  "Failed to make a backup",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (bk backup) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+	var err error
+
+	// determine the target path for the archive
+	var sPath string
+	if !bk.StagingOnly {
+		// regular mode: create a temporary staging directory
+		logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
+		sPath, err = dis.NewSnapshotStagingDir("")
+		if err != nil {
+			return errSnapshotFailed.Wrap(err)
+		}
+		defer func() {
+			logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
+			os.RemoveAll(sPath)
+		}()
+	} else {
+		// staging mode: use dest as a destination
+		sPath = bk.Positionals.Dest
+		if sPath == "" {
+			sPath, err = dis.NewSnapshotStagingDir("")
+			if err != nil {
+				return errSnapshotFailed.Wrap(err)
+			}
+		}
+
+		// create the directory (if it doesn't already exist)
+		logging.LogMessage(context.IOStream, "Creating staging directory")
+		err = os.Mkdir(sPath, fs.ModePerm)
+		if !os.IsExist(err) && err != nil {
+			return errSnapshotFailed.WithMessageF(err)
+		}
+		err = nil
+	}
+	context.Println(sPath)
+
+	logging.LogOperation(func() error {
+		// take a snapshot into the staging area!
+		backup := dis.Backup(context.IOStream, env.BackupDescription{
+			Dest: sPath,
+		})
+
+		// write out the report, ignoring any errors!
+		backup.WriteReport(context.IOStream)
+
+		return nil
+	}, context.IOStream, "Generating Backup")
+
+	// if we requested to only have a staging area, then we are done
+	if bk.StagingOnly {
+		context.Printf("Wrote %s\n", sPath)
+		return nil
+	}
+
+	// create the archive path
+	archivePath := bk.Positionals.Dest
+	if archivePath == "" {
+		archivePath = dis.NewSnapshotArchivePath("")
+	}
+
+	// and write everything into it!
+	// TODO: Should we move the open call to here?
+	var count int64
+	if err := logging.LogOperation(func() error {
+		context.IOStream.Println(archivePath)
+
+		count, err = targz.Package(archivePath, sPath, func(dst, src string) {
+			context.Println(dst)
+		})
+		return err
+	}, context.IOStream, "Writing backup archive"); err != nil {
+		return errSnapshotFailed.Wrap(err)
+	}
+	context.Printf("Wrote %d byte(s) to %s\n", count, archivePath)
+
+	return nil
+}

cmd/snapshot.go

@@ -1,9 +1,8 @@
 package cmd
 
 import (
-	"fmt"
+	"io/fs"
 	"os"
-	"path/filepath"
 
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/env"
@@ -16,7 +15,8 @@ import (
 var Snapshot wisski_distillery.Command = snapshot{}
 
 type snapshot struct {
-	Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"`
+	Keepalive   bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"`
+	StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a snapshot archive, but only create a staging directory"`
 
 	Positionals struct {
 		Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"`
@@ -46,65 +46,81 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
 		return err
 	}
 
-	// TODO: Allow skipping backups of individual parts and make them concurrent!
-
-	// start the snapshot and shutdown the instance (if requested)
 	logging.LogMessage(context.IOStream, "Creating snapshot of instance %s", bi.Positionals.Slug)
 
-	// create a new temporary directory
+	// determine the target path for the archive
-	logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
+	var sPath string
-	sPath, err := dis.NewSnapshotStagingDir(instance.Slug)
+	if !bi.StagingOnly {
-	if err != nil {
+		// regular mode: create a temporary staging directory
-		return errSnapshotFailed.Wrap(err)
+		logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
+		sPath, err = dis.NewSnapshotStagingDir(instance.Slug)
+		if err != nil {
+			return errSnapshotFailed.Wrap(err)
+		}
+		defer func() {
+			logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
+			os.RemoveAll(sPath)
+		}()
+	} else {
+		// staging mode: use dest as a destination
+		sPath = bi.Positionals.Dest
+		if sPath == "" {
+			sPath, err = dis.NewSnapshotStagingDir(instance.Slug)
+			if err != nil {
+				return errSnapshotFailed.Wrap(err)
+			}
+		}
+
+		// create the directory (if it doesn't already exist)
+		logging.LogMessage(context.IOStream, "Creating staging directory")
+		err = os.Mkdir(sPath, fs.ModePerm)
+		if !os.IsExist(err) && err != nil {
+			return errSnapshotFailed.WithMessageF(err)
+		}
+		err = nil
 	}
-	defer func() {
+	context.Println(sPath)
-		logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
+
-		os.RemoveAll(sPath)
+	// TODO: Allow skipping backups of individual parts and make them concurrent!
-	}()
 
 	// take a snapshot into the staging area!
-	sreport := instance.Snapshot(context.IOStream, bi.Keepalive, sPath)
-
-	// write out the report!
 	logging.LogOperation(func() error {
+		sreport := instance.Snapshot(context.IOStream, env.SnapshotDescription{
+			Dest:      sPath,
+			Keepalive: bi.Keepalive,
+		})
 
-		logging.LogOperation(func() error {
+		// write out the report, ignoring any errors!
-			reportPath := filepath.Join(sPath, "report.txt")
+		sreport.WriteReport(context.IOStream)
-			context.Println(reportPath)
-
-			// create the report file!
-			report, err := os.Create(reportPath)
-			if err != nil {
-				return err
-			}
-			defer report.Close()
-
-			// print the report into it!
-			_, err = fmt.Fprintf(report, "%#v\n", sreport)
-			return err
-		}, context.IOStream, "Writing snapshot report")
 
 		return nil
-	}, context.IOStream, "Creating snapshot")
+	}, context.IOStream, "Generating Snapshot")
 
-	// copy everything into the final archive
+	// if we requested to only have a staging area, then we are done
-
+	if bi.StagingOnly {
-	finalPath := bi.Positionals.Dest
+		context.Printf("Wrote %s\n", sPath)
-	if finalPath == "" {
+		return nil
-		finalPath = dis.NewSnapshotArchivePath(instance.Slug)
 	}
 
-	if err := logging.LogOperation(func() error {
+	// create the archive path
-		context.IOStream.Println(finalPath)
+	archivePath := bi.Positionals.Dest
+	if archivePath == "" {
+		archivePath = dis.NewSnapshotArchivePath(instance.Slug)
+	}
 
-		targz.Package(finalPath, sPath, func(src string) {
+	// and write everything into it!
-			context.Println(src)
+	// TODO: Should we move the open call to here?
+	var count int64
+	if err := logging.LogOperation(func() error {
+		context.IOStream.Println(archivePath)
+
+		count, err = targz.Package(archivePath, sPath, func(dst, src string) {
+			context.Println(dst)
 		})
 		return err
-	}, context.IOStream, "Writing final backup"); err != nil {
+	}, context.IOStream, "Writing snapshot archive"); err != nil {
 		return errSnapshotFailed.Wrap(err)
 	}
-	context.Printf("Wrote %s\n", finalPath)
+	context.Printf("Wrote %d byte(s) to %s\n", count, archivePath)
-
 	return nil
 }

cmd/wdcli/main.go

@@ -45,7 +45,7 @@ func init() {
 
 	// backup & cron
 	wdcli.Register(cmd.Snapshot)
-	// wdcli.Register(cmd.BackupAll)
+	wdcli.Register(cmd.Backup)
 	wdcli.Register(cmd.Cron)
 }
 

distillery/backup_all.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-
-
-log_info " => Starting backup process. This might take a while. "
-wait_for_sql
-
-BACKUP_SLUG="$(date +%Y%m%dT%H%M%S)-$(randompw)"
-BACKUP_INSTANCE_DIR="$DEPLOY_BACKUP_INPROGRESS_DIR/$BACKUP_SLUG"
-BACKUP_FINAL_FILE="$DEPLOY_BACKUP_FINAL_DIR/$BACKUP_SLUG.tar.gz"
-
-BACKUP_SQL_FILE="$BACKUP_INSTANCE_DIR/backup.sql"
-
-BACKUP_TRIPLESTORE_DIR="$BACKUP_INSTANCE_DIR/triplestore"
-BACKUP_TRIPLESTORE_SYSTEM="$BACKUP_TRIPLESTORE_DIR/system.nq"
-
-BACKUP_FILESYSTEM_DIR="$BACKUP_INSTANCE_DIR/instances"
-
-# create the backup directories
-log_info " => Making '$BACKUP_INSTANCE_DIR'"
-mkdir -p "$BACKUP_INSTANCE_DIR"
-mkdir -p "$DEPLOY_BACKUP_FINAL_DIR"
-
-function backup_everything() {
-    # backup the configuration
-    log_info " => Backing up configuration"
-    cp "$CONFIG_FILE" "$BACKUP_INSTANCE_DIR/.env" || true
-
-    # Backup sql (complete)
-    log_info " => Backing up the SQL database"
-    dockerized_mysqldump --all-databases > "$BACKUP_SQL_FILE" || true
-
-    # Backup triplestore (complete)
-    log_info " => Backing up Triplestore System"
-    mkdir -p "$BACKUP_TRIPLESTORE_DIR"
-    curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/SYSTEM/statements?infer=false" > "$BACKUP_TRIPLESTORE_SYSTEM" || true
-
-    # backup triplestore (individual)
-    for REPO in `grep -oP '(?<=#repositoryID> ")[^"]+' $BACKUP_TRIPLESTORE_SYSTEM`; do
-        log_info " => Backing up Triplestore Repository '$REPO'"
-        curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/$REPO/statements?infer=false" > "$BACKUP_TRIPLESTORE_DIR/repo_$REPO.nq" || true
-    done
-
-    # backup all the instances
-    log_info "=> Backing up instances"
-    for slug in $(sql_bookkeep_list); do
-        log_info "=> /bin/bash '$DIR/backup_instance.sh' '$slug' '$BACKUP_INSTANCE_DIR/${slug}.tar.gz'"
-        /bin/bash "$DIR/backup_instance.sh" "$slug" "$BACKUP_INSTANCE_DIR/${slug}.tar.gz" 2>&1 | tee "$BACKUP_INSTANCE_DIR/${slug}.log" || true
-    done
-}
-
-# do the entire backup
-backup_everything 2>&1 | tee "$BACKUP_LOG_FILE.log"
-
-# Package the backup into a .tar.gz
-log_info " => Packaging '$BACKUP_FINAL_FILE'"
-pushd "$BACKUP_INSTANCE_DIR" > /dev/null
-tar --totals --checkpoint=10000 -zcf "$BACKUP_FINAL_FILE" .
-popd > /dev/null
-
-# Clean up the unpacked backup
-log_info " => Cleaning up '$BACKUP_INSTANCE_DIR'"
-rm -rf "$BACKUP_INSTANCE_DIR"
-
-log_info " => Removing backups older than $MAX_BACKUP_AGE days"
-find "$DEPLOY_BACKUP_FINAL_DIR" -type f -mtime "+$MAX_BACKUP_AGE" -print -exec rm -f {} \;

env/backup.go

@@ -0,0 +1,189 @@
+package env
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// backupDescription is a description for a backup
+type BackupDescription struct {
+	Dest string // destination path
+}
+
+// Snapshot represents the result of generating a snapshot
+type Backup struct {
+	Description BackupDescription
+
+	// various error states, which are ignored when creating the snapshot
+	ErrPanic interface{}
+
+	SQLErr error
+	TSErr  error
+
+	ConfigFileErr       error
+	ConfigFilesManifest map[string]error
+
+	InstanceListErr   error
+	InstancesManifest []Snapshot
+}
+
+func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
+	backup.Description = description
+
+	// catch anything critical that happened during the snapshot
+	defer func() {
+		backup.ErrPanic = recover()
+	}()
+
+	backup.run(io, dis)
+	return
+}
+
+var errBackupSkipFile = errors.New("<file not found>")
+
+func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
+	// create a wait group, and message channel
+	wg := &sync.WaitGroup{}
+	messages := make(chan string, 4)
+
+	// backup the sql
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
+		messages <- sqlPath
+
+		sql, err := os.Create(sqlPath)
+		if err != nil {
+			backup.SQLErr = err
+			return
+		}
+		defer sql.Close()
+
+		// directly store the result
+		backup.SQLErr = dis.SQL().BackupAll(io, sql)
+	}()
+
+	// backup the triplestore
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		tsPath := filepath.Join(backup.Description.Dest, "triplestore")
+		messages <- tsPath
+
+		// directly store the result
+		backup.TSErr = dis.Triplestore().BackupAll(tsPath)
+	}()
+
+	// backup configuration files
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
+		if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
+			backup.ConfigFileErr = err
+			return
+		}
+
+		files := []string{
+			filepath.Join(dis.Config.DeployRoot, ".env"),  // TODO: put the name of the configuration file somewhere
+			filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
+			dis.Config.SelfOverridesFile,
+			dis.Config.GlobalAuthorizedKeysFile,
+		}
+
+		backup.ConfigFilesManifest = make(map[string]error, len(files))
+		for _, src := range files {
+			if !fsx.IsFile(src) {
+				backup.ConfigFilesManifest[src] = errBackupSkipFile
+				continue
+			}
+			dest := filepath.Join(cfgBackupDir, filepath.Base(src))
+
+			// copy the config file and store the error message
+			messages <- src
+			backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
+		}
+	}()
+
+	// backup instances
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
+		if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		// list all instances
+		instances, err := dis.AllInstances()
+		if err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
+
+		backup.InstancesManifest = make([]Snapshot, len(instances))
+		for i, instance := range instances {
+			backup.InstancesManifest[i] = func() Snapshot {
+				dir := filepath.Join(instancesBackupDir, instance.Slug)
+				if err := os.Mkdir(dir, fs.ModeDir); err != nil {
+					return Snapshot{
+						ErrPanic: err,
+					}
+				}
+
+				messages <- dir
+				return instance.Snapshot(iochild, SnapshotDescription{
+					Dest: dir,
+				})
+			}()
+		}
+
+	}()
+
+	// wait for the group, then close the message channel.
+	go func() {
+		wg.Wait()
+		close(messages)
+	}()
+
+	// print out all the messages
+	for message := range messages {
+		io.Println(message)
+	}
+}
+
+// WriteReport writes out the report belonging to this backup.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (backup Backup) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(backup.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = fmt.Fprintf(report, "%#v\n", backup)
+		return err
+	}, io, "Writing backup report")
+}

env/component_sql.go

@@ -103,7 +103,21 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code"
 func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error {
 	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
 
-	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--database", database)
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}
+
+// BackupAll makes a backup of all sql databases
+func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error {
+	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
+
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
 	if err != nil {
 		return err
 	}

env/component_triplestore.go

@@ -8,6 +8,7 @@ import (
 	"io/fs"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -237,6 +238,62 @@ func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error)
 	return io.Copy(dst, res.Body)
 }
 
+type Repository struct {
+	ID         string `json:"id"`
+	Title      string `json:"title"`
+	URI        string `json:"uri"`
+	Type       string `json:"type"`
+	SesameType string `json:"sesameType"`
+	Location   string `json:"location"`
+	Readable   bool   `json:"readable"`
+	Writable   bool   `json:"writable"`
+	Local      bool   `json:"local"`
+}
+
+func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error) {
+	res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	err = json.NewDecoder(res.Body).Decode(&repos)
+	return
+}
+
+// TriplestoreBackup backs up every graphdb instance into dst
+func (ts TriplestoreComponent) BackupAll(dst string) error {
+	// list all the repositories
+	repos, err := ts.listRepositories()
+	if err != nil {
+		return err
+	}
+
+	// create the base directory
+	if err := os.Mkdir(dst, fs.ModeDir); err != nil {
+		return err
+	}
+
+	// iterate over all the repositories
+	for _, repo := range repos {
+		if rErr := (func(repo Repository) error {
+			name := filepath.Join(dst, repo.ID+".nq")
+
+			dest, err := os.Create(name)
+			if err != nil {
+				return err
+			}
+			defer dest.Close()
+
+			_, err = ts.Backup(dest, repo.ID)
+			return err
+		}(repo)); err == nil && rErr != nil {
+			err = rErr
+		}
+	}
+	return err
+}
+
 var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
 
 func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {

env/snapshot.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
 	"github.com/FAU-CDI/wisski-distillery/internal/password"
@@ -46,6 +47,11 @@ func (dis Distillery) NewSnapshotArchivePath(prefix string) (path string) {
 // The name is guaranteed to be unique within this process.
 func (Distillery) newSnapshotName(prefix string) string {
 	suffix, _ := password.Password(64) // silently ignore any errors!
+	if prefix == "" {
+		prefix = "backup"
+	} else {
+		prefix = "snapshot-" + prefix
+	}
 	return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
 }
 
@@ -62,36 +68,56 @@ func (dis Distillery) NewSnapshotStagingDir(prefix string) (path string, err err
 	return
 }
 
-type SnapshotReport struct {
+// SnapshotDescription is a description for a snapshot
-	Keepalive bool        // was the instance alive while running the snapshot?
+type SnapshotDescription struct {
-	Panic     interface{} // was there a panic?
+	Dest      string // destination path
+	Keepalive bool   // should we keep the instance alive while making the snapshot?
+}
 
-	// errors for the various components of the Snapshot
+// Snapshot represents the result of generating a snapshot
-	StopErr        error
+type Snapshot struct {
-	StartErr       error
+	Description SnapshotDescription
-	BookkeepingErr error
+	Instance    bookkeeping.Instance
-	FilesystemErr  error
+
-	TriplestoreErr error
+	// various error states, which are ignored when creating the snapshot
-	SQLErr         error
+	ErrPanic interface{} // panic, if any
+
+	ErrStart error
+	ErrStop  error
+
+	ErrBookkeep    error
+	ErrFilesystem  error
+	ErrTriplestore error
+	ErrSSQL        error
 }
 
 // Snapshot creates a new snapshot of this instance into dest
-func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest string) (report SnapshotReport) {
+func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
+	// setup the snapshot
+	snapshot.Description = desc
+	snapshot.Instance = instance.Instance
+
 	// catch anything critical that happened during the snapshot
 	defer func() {
-		report.Panic = recover()
+		snapshot.ErrPanic = recover()
 	}()
 
+	// and do the create!
+	snapshot.create(io, instance)
+
+	return
+}
+
+func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 	stack := instance.Stack()
 
 	// stop the instance (unless it was explicitly asked to not do so!)
-	report.Keepalive = keepalive
+	if !snapshot.Description.Keepalive {
-	if !keepalive {
 		logging.LogMessage(io, "Stopping instance")
-		report.StopErr = stack.Down(io)
+		snapshot.ErrStop = stack.Down(io)
 		defer func() {
 			logging.LogMessage(io, "Starting instance")
-			report.StartErr = stack.Up(io)
+			snapshot.ErrStart = stack.Up(io)
 		}()
 	}
 
@@ -104,19 +130,19 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		bkPath := filepath.Join(dest, "bookkeeping.txt")
+		bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
 		messages <- bkPath
 
 		info, err := os.Create(bkPath)
 		if err != nil {
-			report.BookkeepingErr = err
+			snapshot.ErrBookkeep = err
 			return
 		}
 		defer info.Close()
 
 		// print whatever is in the database
 		// TODO: This should be sql code, maybe gorm can do that?
-		_, report.BookkeepingErr = fmt.Fprintf(info, "%#v\n", instance.Instance)
+		_, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance)
 	}()
 
 	// backup the filesystem
@@ -124,14 +150,14 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		fsPath := filepath.Join(dest, filepath.Base(instance.FilesystemBase))
+		fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
 		if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
-			report.FilesystemErr = err
+			snapshot.ErrFilesystem = err
 			return
 		}
 
 		// copy over whatever is in the base directory
-		report.FilesystemErr = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
+		snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
 			messages <- dst
 		})
 	}()
@@ -141,17 +167,17 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		tsPath := filepath.Join(dest, instance.GraphDBRepository+".nq")
+		tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
 		messages <- tsPath
 
 		nquads, err := os.Create(tsPath)
 		if err != nil {
-			report.TriplestoreErr = err
+			snapshot.ErrTriplestore = err
 		}
 		defer nquads.Close()
 
 		// directly store the result
-		_, report.TriplestoreErr = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
+		_, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
 	}()
 
 	// backup the sql database
@@ -159,18 +185,18 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		sqlPath := filepath.Join(dest, instance.SqlDatabase+".sql")
+		sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
 		messages <- sqlPath
 
 		sql, err := os.Create(sqlPath)
 		if err != nil {
-			report.SQLErr = err
+			snapshot.ErrSSQL = err
 			return
 		}
 		defer sql.Close()
 
 		// directly store the result
-		report.SQLErr = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
+		snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
 	}()
 
 	// TODO: Backup the docker image
@@ -185,6 +211,24 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	for message := range messages {
 		io.Println(message)
 	}
-
+}
-	return
+
+// WriteReport writes out the report belonging to this snapshot.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = fmt.Fprintf(report, "%#v\n", snapshot)
+		return err
+	}, io, "Writing snapshot report")
 }

internal/targz/targz.go

@@ -11,11 +11,11 @@ import (
 
 // Package packages the directory src into dst.
 // onCopy, when not nil, is called for each file being copied into the archive.
-func Package(dst, src string, onCopy func(src string)) error {
+func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
 	// create the target archive
 	archive, err := os.Create(dst)
 	if err != nil {
-		return err
+		return 0, err
 	}
 	defer archive.Close()
 
@@ -28,21 +28,28 @@ func Package(dst, src string, onCopy func(src string)) error {
 	defer tarHandle.Close()
 
 	// and walk through it!
-	return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
+	err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
-		if onCopy != nil {
-			onCopy(path)
-		}
-
 		if err != nil {
 			return err
 		}
 
+		// determine the relative path
+		var relpath string
+		relpath, err = filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+
+		if onCopy != nil {
+			onCopy(relpath, path)
+		}
+
 		// create a file info header!
-		tInfo, err := tar.FileInfoHeader(info, path)
+		tInfo, err := tar.FileInfoHeader(info, relpath)
 		if err != nil {
 			return err
 		}
-		tInfo.Name = filepath.ToSlash(path)
+		tInfo.Name = filepath.ToSlash(relpath)
 
 		// write it!
 		if err := tarHandle.WriteHeader(tInfo); err != nil {
@@ -62,8 +69,9 @@ func Package(dst, src string, onCopy func(src string)) error {
 		defer handle.Close()
 
 		// and copy it into the archive
-		_, err = io.Copy(tarHandle, handle)
+		ccount, err := io.Copy(tarHandle, handle)
+		count += ccount
 		return err
 	})
-
+	return
 }