wdcli: Implement backup & snapshot

This commit implements command backup and snapshot.

env/backup.go

@@ -0,0 +1,189 @@
+package env
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// backupDescription is a description for a backup
+type BackupDescription struct {
+	Dest string // destination path
+}
+
+// Snapshot represents the result of generating a snapshot
+type Backup struct {
+	Description BackupDescription
+
+	// various error states, which are ignored when creating the snapshot
+	ErrPanic interface{}
+
+	SQLErr error
+	TSErr  error
+
+	ConfigFileErr       error
+	ConfigFilesManifest map[string]error
+
+	InstanceListErr   error
+	InstancesManifest []Snapshot
+}
+
+func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
+	backup.Description = description
+
+	// catch anything critical that happened during the snapshot
+	defer func() {
+		backup.ErrPanic = recover()
+	}()
+
+	backup.run(io, dis)
+	return
+}
+
+var errBackupSkipFile = errors.New("<file not found>")
+
+func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
+	// create a wait group, and message channel
+	wg := &sync.WaitGroup{}
+	messages := make(chan string, 4)
+
+	// backup the sql
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
+		messages <- sqlPath
+
+		sql, err := os.Create(sqlPath)
+		if err != nil {
+			backup.SQLErr = err
+			return
+		}
+		defer sql.Close()
+
+		// directly store the result
+		backup.SQLErr = dis.SQL().BackupAll(io, sql)
+	}()
+
+	// backup the triplestore
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		tsPath := filepath.Join(backup.Description.Dest, "triplestore")
+		messages <- tsPath
+
+		// directly store the result
+		backup.TSErr = dis.Triplestore().BackupAll(tsPath)
+	}()
+
+	// backup configuration files
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
+		if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
+			backup.ConfigFileErr = err
+			return
+		}
+
+		files := []string{
+			filepath.Join(dis.Config.DeployRoot, ".env"),  // TODO: put the name of the configuration file somewhere
+			filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
+			dis.Config.SelfOverridesFile,
+			dis.Config.GlobalAuthorizedKeysFile,
+		}
+
+		backup.ConfigFilesManifest = make(map[string]error, len(files))
+		for _, src := range files {
+			if !fsx.IsFile(src) {
+				backup.ConfigFilesManifest[src] = errBackupSkipFile
+				continue
+			}
+			dest := filepath.Join(cfgBackupDir, filepath.Base(src))
+
+			// copy the config file and store the error message
+			messages <- src
+			backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
+		}
+	}()
+
+	// backup instances
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
+		if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		// list all instances
+		instances, err := dis.AllInstances()
+		if err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
+
+		backup.InstancesManifest = make([]Snapshot, len(instances))
+		for i, instance := range instances {
+			backup.InstancesManifest[i] = func() Snapshot {
+				dir := filepath.Join(instancesBackupDir, instance.Slug)
+				if err := os.Mkdir(dir, fs.ModeDir); err != nil {
+					return Snapshot{
+						ErrPanic: err,
+					}
+				}
+
+				messages <- dir
+				return instance.Snapshot(iochild, SnapshotDescription{
+					Dest: dir,
+				})
+			}()
+		}
+
+	}()
+
+	// wait for the group, then close the message channel.
+	go func() {
+		wg.Wait()
+		close(messages)
+	}()
+
+	// print out all the messages
+	for message := range messages {
+		io.Println(message)
+	}
+}
+
+// WriteReport writes out the report belonging to this backup.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (backup Backup) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(backup.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = fmt.Fprintf(report, "%#v\n", backup)
+		return err
+	}, io, "Writing backup report")
+}

env/component_sql.go

@@ -103,7 +103,21 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code"
 func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error {
 	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
 
-	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--database", database)
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}
+
+// BackupAll makes a backup of all sql databases
+func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error {
+	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
+
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
 	if err != nil {
 		return err
 	}

env/component_triplestore.go

@@ -8,6 +8,7 @@ import (
 	"io/fs"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -237,6 +238,62 @@ func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error)
 	return io.Copy(dst, res.Body)
 }
 
+type Repository struct {
+	ID         string `json:"id"`
+	Title      string `json:"title"`
+	URI        string `json:"uri"`
+	Type       string `json:"type"`
+	SesameType string `json:"sesameType"`
+	Location   string `json:"location"`
+	Readable   bool   `json:"readable"`
+	Writable   bool   `json:"writable"`
+	Local      bool   `json:"local"`
+}
+
+func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error) {
+	res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	err = json.NewDecoder(res.Body).Decode(&repos)
+	return
+}
+
+// TriplestoreBackup backs up every graphdb instance into dst
+func (ts TriplestoreComponent) BackupAll(dst string) error {
+	// list all the repositories
+	repos, err := ts.listRepositories()
+	if err != nil {
+		return err
+	}
+
+	// create the base directory
+	if err := os.Mkdir(dst, fs.ModeDir); err != nil {
+		return err
+	}
+
+	// iterate over all the repositories
+	for _, repo := range repos {
+		if rErr := (func(repo Repository) error {
+			name := filepath.Join(dst, repo.ID+".nq")
+
+			dest, err := os.Create(name)
+			if err != nil {
+				return err
+			}
+			defer dest.Close()
+
+			_, err = ts.Backup(dest, repo.ID)
+			return err
+		}(repo)); err == nil && rErr != nil {
+			err = rErr
+		}
+	}
+	return err
+}
+
 var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
 
 func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {

env/snapshot.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
 	"github.com/FAU-CDI/wisski-distillery/internal/password"
@@ -46,6 +47,11 @@ func (dis Distillery) NewSnapshotArchivePath(prefix string) (path string) {
 // The name is guaranteed to be unique within this process.
 func (Distillery) newSnapshotName(prefix string) string {
 	suffix, _ := password.Password(64) // silently ignore any errors!
+	if prefix == "" {
+		prefix = "backup"
+	} else {
+		prefix = "snapshot-" + prefix
+	}
 	return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
 }
 
@@ -62,36 +68,56 @@ func (dis Distillery) NewSnapshotStagingDir(prefix string) (path string, err err
 	return
 }
 
-type SnapshotReport struct {
-	Keepalive bool        // was the instance alive while running the snapshot?
-	Panic     interface{} // was there a panic?
+// SnapshotDescription is a description for a snapshot
+type SnapshotDescription struct {
+	Dest      string // destination path
+	Keepalive bool   // should we keep the instance alive while making the snapshot?
+}
 
-	// errors for the various components of the Snapshot
-	StopErr        error
-	StartErr       error
-	BookkeepingErr error
-	FilesystemErr  error
-	TriplestoreErr error
-	SQLErr         error
+// Snapshot represents the result of generating a snapshot
+type Snapshot struct {
+	Description SnapshotDescription
+	Instance    bookkeeping.Instance
+
+	// various error states, which are ignored when creating the snapshot
+	ErrPanic interface{} // panic, if any
+
+	ErrStart error
+	ErrStop  error
+
+	ErrBookkeep    error
+	ErrFilesystem  error
+	ErrTriplestore error
+	ErrSSQL        error
 }
 
 // Snapshot creates a new snapshot of this instance into dest
-func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest string) (report SnapshotReport) {
+func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
+	// setup the snapshot
+	snapshot.Description = desc
+	snapshot.Instance = instance.Instance
+
 	// catch anything critical that happened during the snapshot
 	defer func() {
-		report.Panic = recover()
+		snapshot.ErrPanic = recover()
 	}()
 
+	// and do the create!
+	snapshot.create(io, instance)
+
+	return
+}
+
+func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 	stack := instance.Stack()
 
 	// stop the instance (unless it was explicitly asked to not do so!)
-	report.Keepalive = keepalive
-	if !keepalive {
+	if !snapshot.Description.Keepalive {
 		logging.LogMessage(io, "Stopping instance")
-		report.StopErr = stack.Down(io)
+		snapshot.ErrStop = stack.Down(io)
 		defer func() {
 			logging.LogMessage(io, "Starting instance")
-			report.StartErr = stack.Up(io)
+			snapshot.ErrStart = stack.Up(io)
 		}()
 	}
 
@@ -104,19 +130,19 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		bkPath := filepath.Join(dest, "bookkeeping.txt")
+		bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
 		messages <- bkPath
 
 		info, err := os.Create(bkPath)
 		if err != nil {
-			report.BookkeepingErr = err
+			snapshot.ErrBookkeep = err
 			return
 		}
 		defer info.Close()
 
 		// print whatever is in the database
 		// TODO: This should be sql code, maybe gorm can do that?
-		_, report.BookkeepingErr = fmt.Fprintf(info, "%#v\n", instance.Instance)
+		_, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance)
 	}()
 
 	// backup the filesystem
@@ -124,14 +150,14 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		fsPath := filepath.Join(dest, filepath.Base(instance.FilesystemBase))
+		fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
 		if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
-			report.FilesystemErr = err
+			snapshot.ErrFilesystem = err
 			return
 		}
 
 		// copy over whatever is in the base directory
-		report.FilesystemErr = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
+		snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
 			messages <- dst
 		})
 	}()
@@ -141,17 +167,17 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		tsPath := filepath.Join(dest, instance.GraphDBRepository+".nq")
+		tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
 		messages <- tsPath
 
 		nquads, err := os.Create(tsPath)
 		if err != nil {
-			report.TriplestoreErr = err
+			snapshot.ErrTriplestore = err
 		}
 		defer nquads.Close()
 
 		// directly store the result
-		_, report.TriplestoreErr = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
+		_, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
 	}()
 
 	// backup the sql database
@@ -159,18 +185,18 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	go func() {
 		defer wg.Done()
 
-		sqlPath := filepath.Join(dest, instance.SqlDatabase+".sql")
+		sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
 		messages <- sqlPath
 
 		sql, err := os.Create(sqlPath)
 		if err != nil {
-			report.SQLErr = err
+			snapshot.ErrSSQL = err
 			return
 		}
 		defer sql.Close()
 
 		// directly store the result
-		report.SQLErr = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
+		snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
 	}()
 
 	// TODO: Backup the docker image
@@ -185,6 +211,24 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
 	for message := range messages {
 		io.Println(message)
 	}
-
-	return
+}
+
+// WriteReport writes out the report belonging to this snapshot.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = fmt.Fprintf(report, "%#v\n", snapshot)
+		return err
+	}, io, "Writing snapshot report")
 }