{backup,snapshort}: Improve behaviour

This commit improves the behaviour of 'backup' and 'snapshot' by
treating symbolic links properly, as well as writes proper reports.

cmd/backup.go

@@ -112,8 +112,9 @@ func (bk backup) Run(context wisski_distillery.Context) error {
 		context.IOStream.Println(archivePath)
 
 		count, err = targz.Package(archivePath, sPath, func(dst, src string) {
-			context.Println(dst)
+			context.Printf("\033[2K\r%s", dst)
 		})
+		context.Println("")
 		return err
 	}, context.IOStream, "Writing backup archive"); err != nil {
 		return errSnapshotFailed.Wrap(err)

cmd/snapshot.go

@@ -116,8 +116,9 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
 		context.IOStream.Println(archivePath)
 
 		count, err = targz.Package(archivePath, sPath, func(dst, src string) {
-			context.Println(dst)
+			context.Printf("\033[2K\r%s", dst)
 		})
+		context.Println("")
 		return err
 	}, context.IOStream, "Writing snapshot archive"); err != nil {
 		return errSnapshotFailed.Wrap(err)

internal/env/backup.go

@@ -1,17 +1,22 @@
 package env
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
 )
 
 // backupDescription is a description for a backup
@@ -23,17 +28,75 @@ type BackupDescription struct {
 type Backup struct {
 	Description BackupDescription
 
+	// Start and End Time of the backup
+	StartTime time.Time
+	EndTime   time.Time
+
 	// various error states, which are ignored when creating the snapshot
 	ErrPanic interface{}
 
+	// SQL and triplestore errors
 	SQLErr error
 	TSErr  error
 
+	// TODO: Make this proper
 	ConfigFileErr       error
 	ConfigFilesManifest map[string]error
 
+	// Snapshots containing instances
 	InstanceListErr   error
-	InstancesManifest []Snapshot
+	InstanceSnapshots []Snapshot
+
+	// List of files included
+	Manifest []string
+}
+
+func (backup Backup) String() string {
+	var builder strings.Builder
+	backup.Report(&builder)
+	return builder.String()
+}
+
+// Report writes a report from backup into w
+func (backup Backup) Report(w io.Writer) {
+	// TODO: Errors
+	encoder := json.NewEncoder(w)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(w, "======= Backup =======\n")
+
+	fmt.Fprintf(w, "Start: %s\n", backup.StartTime)
+	fmt.Fprintf(w, "End:   %s\n", backup.EndTime)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Description =======\n")
+	encoder.Encode(backup.Description)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Errors =======\n")
+	fmt.Fprintf(w, "Panic:           %v\n", backup.ErrPanic)
+	fmt.Fprintf(w, "SQLErr:          %s\n", backup.SQLErr)
+	fmt.Fprintf(w, "TSErr:           %s\n", backup.TSErr)
+	fmt.Fprintf(w, "ConfigFileErr:   %s\n", backup.ConfigFileErr)
+	fmt.Fprintf(w, "InstanceListErr: %s\n", backup.InstanceListErr)
+
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Config Files =======\n")
+	encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
+
+	io.WriteString(w, "======= Snapshots =======\n")
+	for _, s := range backup.InstanceSnapshots {
+		io.WriteString(w, s.String())
+		io.WriteString(w, "\n")
+	}
+
+	io.WriteString(w, "======= Manifest =======\n")
+	for _, file := range backup.Manifest {
+		io.WriteString(w, file+"\n")
+	}
+
+	io.WriteString(w, "\n")
 }
 
 func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
@@ -44,7 +107,15 @@ func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription)
 		backup.ErrPanic = recover()
 	}()
 
-	backup.run(io, dis)
+	// do the create keeping track of time!
+	logging.LogOperation(func() error {
+		backup.StartTime = time.Now()
+		backup.run(io, dis)
+		backup.EndTime = time.Now()
+
+		return nil
+	}, io, "Writing backup files")
+
 	return
 }
 
@@ -53,7 +124,7 @@ var errBackupSkipFile = errors.New("<file not found>")
 func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 	// create a wait group, and message channel
 	wg := &sync.WaitGroup{}
-	messages := make(chan string, 4)
+	files := make(chan string, 4)
 
 	// backup the sql
 	wg.Add(1)
@@ -61,7 +132,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 		defer wg.Done()
 
 		sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
-		messages <- sqlPath
+		files <- sqlPath
 
 		sql, err := os.Create(sqlPath)
 		if err != nil {
@@ -80,7 +151,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 		defer wg.Done()
 
 		tsPath := filepath.Join(backup.Description.Dest, "triplestore")
-		messages <- tsPath
+		files <- tsPath
 
 		// directly store the result
 		backup.TSErr = dis.Triplestore().BackupAll(tsPath)
@@ -97,15 +168,15 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 			return
 		}
 
-		files := []string{
+		configs := []string{
-			filepath.Join(dis.Config.DeployRoot, ".env"),  // TODO: put the name of the configuration file somewhere
+			dis.Config.ConfigPath,
-			filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
+			filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
 			dis.Config.SelfOverridesFile,
 			dis.Config.GlobalAuthorizedKeysFile,
 		}
 
-		backup.ConfigFilesManifest = make(map[string]error, len(files))
+		backup.ConfigFilesManifest = make(map[string]error, len(configs))
-		for _, src := range files {
+		for _, src := range configs {
 			if !fsx.IsFile(src) {
 				backup.ConfigFilesManifest[src] = errBackupSkipFile
 				continue
@@ -113,7 +184,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 			dest := filepath.Join(cfgBackupDir, filepath.Base(src))
 
 			// copy the config file and store the error message
-			messages <- src
+			files <- src
 			backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
 		}
 	}()
@@ -138,9 +209,9 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 
 		iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
 
-		backup.InstancesManifest = make([]Snapshot, len(instances))
+		backup.InstanceSnapshots = make([]Snapshot, len(instances))
 		for i, instance := range instances {
-			backup.InstancesManifest[i] = func() Snapshot {
+			backup.InstanceSnapshots[i] = func() Snapshot {
 				dir := filepath.Join(instancesBackupDir, instance.Slug)
 				if err := os.Mkdir(dir, fs.ModeDir); err != nil {
 					return Snapshot{
@@ -148,7 +219,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 					}
 				}
 
-				messages <- dir
+				files <- dir
 				return instance.Snapshot(iochild, SnapshotDescription{
 					Dest: dir,
 				})
@@ -160,13 +231,29 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
 	// wait for the group, then close the message channel.
 	go func() {
 		wg.Wait()
-		close(messages)
+		close(files)
 	}()
 
-	// print out all the messages
+	for file := range files {
-	for message := range messages {
+		// get the relative path to the root of the manifest.
-		io.Println(message)
+		// nothing *should* go wrong, but in case it does, use the original path.
+		path, err := filepath.Rel(backup.Description.Dest, file)
+		if err != nil {
+			path = file
+		}
+
+		// write it to the command line
+		// and also add it to the manifest
+		io.Printf("\033[2K\r%s", path)
+		backup.Manifest = append(backup.Manifest, path)
 	}
+	slices.Sort(backup.Manifest) // backup the manifest
+	io.Println("")
+
+	// sort the instances manifest
+	slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
+		return a.Instance.Slug < b.Instance.Slug
+	})
 }
 
 // WriteReport writes out the report belonging to this backup.
@@ -184,7 +271,7 @@ func (backup Backup) WriteReport(io stream.IOStream) error {
 		defer report.Close()
 
 		// print the report into it!
-		_, err = fmt.Fprintf(report, "%#v\n", backup)
+		_, err = report.WriteString(backup.String())
 		return err
 	}, io, "Writing backup report")
 }

internal/env/snapshot.go

@@ -1,10 +1,13 @@
 package env
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -13,6 +16,7 @@ import (
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/password"
 	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
 )
 
 // SnapshotsDir returns the path that contains all snapshot related data.
@@ -79,17 +83,76 @@ type Snapshot struct {
 	Description SnapshotDescription
 	Instance    bookkeeping.Instance
 
-	// various error states, which are ignored when creating the snapshot
+	// Start and End Time of the snapshot
-	ErrPanic interface{} // panic, if any
+	StartTime time.Time
+	EndTime   time.Time
 
+	// Generic Panic that may have occured
+	ErrPanic interface{}
+
+	// Errors during starting and stopping the system
 	ErrStart error
 	ErrStop  error
 
+	// List of files included
+	Manifest []string
+
+	// Errors during other parts
 	ErrBookkeep    error
 	ErrPathbuilder error
 	ErrFilesystem  error
 	ErrTriplestore error
-	ErrSSQL        error
+	ErrSQL         error
+}
+
+func (snapshot Snapshot) String() string {
+	var builder strings.Builder
+	snapshot.Report(&builder)
+	return builder.String()
+}
+
+// Report writes a report from snapshot into w
+func (snapshot Snapshot) Report(w io.Writer) {
+	// TODO: Errors of the writer!
+	encoder := json.NewEncoder(w)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(w, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
+
+	fmt.Fprintf(w, "Slug:  %s\n", snapshot.Instance.Slug)
+	fmt.Fprintf(w, "Dest:  %s\n", snapshot.Description.Dest)
+
+	fmt.Fprintf(w, "Start: %s\n", snapshot.StartTime)
+	fmt.Fprintf(w, "End:   %s\n", snapshot.EndTime)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Description =======\n")
+	encoder.Encode(snapshot.Description)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Instance =======\n")
+	encoder.Encode(snapshot.Instance)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Errors =======\n")
+	fmt.Fprintf(w, "Panic:       %v\n", snapshot.ErrPanic)
+	fmt.Fprintf(w, "Start:       %s\n", snapshot.ErrStart)
+	fmt.Fprintf(w, "Stop:        %s\n", snapshot.ErrStop)
+	fmt.Fprintf(w, "Bookkeep:    %s\n", snapshot.ErrBookkeep)
+	fmt.Fprintf(w, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
+	fmt.Fprintf(w, "Filesystem:  %s\n", snapshot.ErrFilesystem)
+	fmt.Fprintf(w, "Triplestore: %s\n", snapshot.ErrTriplestore)
+	fmt.Fprintf(w, "SQL:         %s\n", snapshot.ErrSQL)
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= Manifest =======\n")
+	for _, file := range snapshot.Manifest {
+		io.WriteString(w, file+"\n")
+	}
+
+	io.WriteString(w, "\n")
+
+	io.WriteString(w, "======= End Snapshot Report "+snapshot.Instance.Slug+" =======\n")
 }
 
 // Snapshot creates a new snapshot of this instance into dest
@@ -98,13 +161,19 @@ func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription)
 	snapshot.Description = desc
 	snapshot.Instance = instance.Instance
 
-	// catch anything critical that happened during the snapshot
+	// capture anything critical, and write the end time
 	defer func() {
 		snapshot.ErrPanic = recover()
 	}()
 
-	// and do the create!
+	// do the create keeping track of time!
-	snapshot.create(io, instance)
+	logging.LogOperation(func() error {
+		snapshot.StartTime = time.Now()
+		snapshot.create(io, instance)
+		snapshot.EndTime = time.Now()
+
+		return nil
+	}, io, "Writing snapshot files")
 
 	return
 }
@@ -124,7 +193,7 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 
 	// create a wait group, and message channel
 	wg := &sync.WaitGroup{}
-	messages := make(chan string, 4)
+	files := make(chan string, 4)
 
 	// write bookkeeping information
 	wg.Add(1)
@@ -132,7 +201,7 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 		defer wg.Done()
 
 		bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
-		messages <- bkPath
+		files <- bkPath
 
 		info, err := os.Create(bkPath)
 		if err != nil {
@@ -147,12 +216,13 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 	}()
 
 	// write pathbuilders
+	// TODO: Move this outside of the up/down stuff!
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 
 		pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
-		messages <- pbPath
+		files <- pbPath
 
 		// create the directory!
 		if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
@@ -170,14 +240,10 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 		defer wg.Done()
 
 		fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
-		if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
-			snapshot.ErrFilesystem = err
-			return
-		}
 
 		// copy over whatever is in the base directory
 		snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
-			messages <- dst
+			files <- dst
 		})
 	}()
 
@@ -187,11 +253,12 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 		defer wg.Done()
 
 		tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
-		messages <- tsPath
+		files <- tsPath
 
 		nquads, err := os.Create(tsPath)
 		if err != nil {
 			snapshot.ErrTriplestore = err
+			return
 		}
 		defer nquads.Close()
 
@@ -205,17 +272,17 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 		defer wg.Done()
 
 		sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
-		messages <- sqlPath
+		files <- sqlPath
 
 		sql, err := os.Create(sqlPath)
 		if err != nil {
-			snapshot.ErrSSQL = err
+			snapshot.ErrSQL = err
 			return
 		}
 		defer sql.Close()
 
 		// directly store the result
-		snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
+		snapshot.ErrSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
 	}()
 
 	// TODO: Backup the docker image
@@ -223,13 +290,26 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
 	// wait for the group, then close the message channel.
 	go func() {
 		wg.Wait()
-		close(messages)
+		close(files)
 	}()
 
-	// print out all the messages
+	for file := range files {
-	for message := range messages {
+		// get the relative path to the root of the manifest.
-		io.Println(message)
+		// nothing *should* go wrong, but in case it does, use the original path.
+		path, err := filepath.Rel(snapshot.Description.Dest, file)
+		if err != nil {
+			path = file
+		}
+
+		// write it to the command line
+		// and also add it to the manifest
+		io.Printf("\033[2K\r%s", path)
+		snapshot.Manifest = append(snapshot.Manifest, path)
 	}
+	io.Println("")
+
+	// make sure the manifest is sorted!
+	slices.Sort(snapshot.Manifest)
 }
 
 // WriteReport writes out the report belonging to this snapshot.
@@ -247,7 +327,7 @@ func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
 		defer report.Close()
 
 		// print the report into it!
-		_, err = fmt.Fprintf(report, "%#v\n", snapshot)
+		_, err = report.WriteString(snapshot.String())
 		return err
 	}, io, "Writing snapshot report")
 }

pkg/fsx/copy.go

@@ -3,13 +3,16 @@ package fsx
 import (
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 )
 
-var ErrCopySameFile = errors.New("src and dst must be different files")
+var ErrCopySameFile = errors.New("src and dst must be different")
 
 // CopyFile copies a file from src to dst.
+// When src points to a symbolic link, will copy the symbolic link.
+//
 // When dst and src are the same file, returns ErrCopySameFile.
 func CopyFile(dst, src string) error {
 	if SameFile(src, dst) {
@@ -41,10 +44,37 @@ func CopyFile(dst, src string) error {
 	return err
 }
 
-var ErrCopyNoDirectory = errors.New("dst is not a directory")
+// CopyLink copies a link from src to dst.
+// If dst already exists, it is deleted and then re-created.
+func CopyLink(dst, src string) error {
+	// if they're the same file that is an error
+	if SameFile(dst, src) {
+		return ErrCopySameFile
+	}
+
+	// read the link target
+	target, err := os.Readlink(src)
+	if err != nil {
+		return err
+	}
+
+	// delete it if it already exists
+	if Exists(dst) {
+		if err := os.Remove(dst); err != nil {
+			return err
+		}
+	}
+
+	// make the symbolic link!
+	return os.Symlink(target, dst)
+}
+
+var ErrDstFile = errors.New("dst is a file")
 
 // CopyDirectory copies the directory src to dst recursively.
-// The destination directory must exist, or an error is returned.
+//
+// Existing files and directories are overwritten.
+// When a directory already exists, additional files are not deleted.
 //
 // onCopy, when not nil, is called for each file or directory being copied.
 func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
@@ -52,56 +82,51 @@ func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
 	if SameFile(src, dst) {
 		return ErrCopySameFile
 	}
-	if !IsDirectory(dst) {
+	if IsFile(dst) {
-		return ErrCopyNoDirectory
+		return ErrDstFile
 	}
 
-	// call onCopy for this directory!
+	return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
-	if onCopy != nil {
-		onCopy(dst, src)
-	}
-
-	// iterate over the entries or bail out
-	entries, err := os.ReadDir(src)
-	if err != nil {
-		return err
-	}
-
-	for _, entry := range entries {
-		name := entry.Name()
-		eDest := filepath.Join(dst, name)
-		eSrc := filepath.Join(src, name)
-
-		// it is not a directory => Use CopyFile
-		if !entry.IsDir() {
-			if onCopy != nil {
-				onCopy(eDest, eSrc)
-			}
-
-			// do the copy!
-			if err := CopyFile(eDest, eSrc); err != nil {
-				return err
-			}
-
-			continue
-		}
-
-		// find out the mode of the entry
-		eInfo, err := entry.Info()
 		if err != nil {
 			return err
 		}
 
-		// make the target directory
+		// determine the real target path
-		if err := os.Mkdir(eDest, eInfo.Mode()); err != nil {
+		var relpath string
+		relpath, err = filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		dst := filepath.Join(dst, relpath)
+
+		// call the hook
+		if onCopy != nil {
+			onCopy(dst, src)
+		}
+
+		// stat the directory, so that we can get mode, and info later!
+		info, err := d.Info()
+		if err != nil {
 			return err
 		}
 
-		// do the copy!
+		// if we have a symbolic link, copy the link!
-		if err := CopyDirectory(eDest, eSrc, onCopy); err != nil {
+		if info.Mode()&os.ModeSymlink != 0 {
-			return err
+			return CopyLink(dst, path)
 		}
-	}
 
-	return nil
+		// if we got a file, we should copy it normally
+		if !d.IsDir() {
+			return CopyFile(dst, path)
+		}
+
+		// create the directory, but ignore an error if the directory already exists.
+		// this is so that we can copy one tree into another tree.
+		err = os.Mkdir(dst, info.Mode())
+		if os.IsExist(err) && IsDirectory(dst) {
+			err = nil
+		}
+
+		return err
+	})
 }

pkg/fsx/type.go

@@ -7,7 +7,7 @@ import (
 
 // Exists checks if the given path exists
 func Exists(path string) bool {
-	_, err := os.Stat(path)
+	_, err := os.Lstat(path)
 	return err == nil
 }
 
@@ -22,3 +22,9 @@ func IsFile(path string) bool {
 	info, err := os.Stat(path)
 	return err == nil && info.Mode().IsRegular()
 }
+
+// IsLink checks if the provided path exists and is a symlink
+func IsLink(path string) bool {
+	info, err := os.Lstat(path)
+	return err == nil && info.Mode()&os.ModeSymlink != 0
+}

pkg/targz/targz.go

@@ -31,7 +31,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64,
 	defer tarHandle.Close()
 
 	// and walk through it!
-	err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
+	err = filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -43,10 +43,17 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64,
 			return err
 		}
 
+		// call the oncopy!
 		if onCopy != nil {
 			onCopy(relpath, path)
 		}
 
+		// read mode etc
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
 		// create a file info header!
 		tInfo, err := tar.FileInfoHeader(info, relpath)
 		if err != nil {
@@ -60,7 +67,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64,
 		}
 
 		// a directory => no more writing required
-		if info.IsDir() {
+		if entry.IsDir() {
 			return nil
 		}