From a4f91ae7cf699523e0e559dbdc1f4db971942572 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Tue, 13 Sep 2022 11:44:32 +0200 Subject: [PATCH] {backup,snapshort}: Improve behaviour This commit improves the behaviour of 'backup' and 'snapshot' by treating symbolic links properly, as well as writes proper reports. --- cmd/backup.go | 3 +- cmd/snapshot.go | 3 +- internal/env/backup.go | 125 ++++++++++++++++++++++++++++++++------ internal/env/snapshot.go | 126 ++++++++++++++++++++++++++++++++------- pkg/fsx/copy.go | 113 +++++++++++++++++++++-------------- pkg/fsx/type.go | 8 ++- pkg/targz/targz.go | 11 +++- 7 files changed, 298 insertions(+), 91 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index e02d65e..0e30929 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -112,8 +112,9 @@ func (bk backup) Run(context wisski_distillery.Context) error { context.IOStream.Println(archivePath) count, err = targz.Package(archivePath, sPath, func(dst, src string) { - context.Println(dst) + context.Printf("\033[2K\r%s", dst) }) + context.Println("") return err }, context.IOStream, "Writing backup archive"); err != nil { return errSnapshotFailed.Wrap(err) diff --git a/cmd/snapshot.go b/cmd/snapshot.go index f48b6fc..41a767c 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -116,8 +116,9 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { context.IOStream.Println(archivePath) count, err = targz.Package(archivePath, sPath, func(dst, src string) { - context.Println(dst) + context.Printf("\033[2K\r%s", dst) }) + context.Println("") return err }, context.IOStream, "Writing snapshot archive"); err != nil { return errSnapshotFailed.Wrap(err) diff --git a/internal/env/backup.go b/internal/env/backup.go index 3aca5ac..a4b8b28 100644 --- a/internal/env/backup.go +++ b/internal/env/backup.go @@ -1,17 +1,22 @@ package env import ( + "encoding/json" "fmt" + "io" "io/fs" "os" "path/filepath" + "strings" "sync" "time" + "github.com/FAU-CDI/wisski-distillery/internal/core" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/pkg/errors" "github.com/tkw1536/goprogram/stream" + "golang.org/x/exp/slices" ) // backupDescription is a description for a backup @@ -23,17 +28,75 @@ type BackupDescription struct { type Backup struct { Description BackupDescription + // Start and End Time of the backup + StartTime time.Time + EndTime time.Time + // various error states, which are ignored when creating the snapshot ErrPanic interface{} + // SQL and triplestore errors SQLErr error TSErr error + // TODO: Make this proper ConfigFileErr error ConfigFilesManifest map[string]error + // Snapshots containing instances InstanceListErr error - InstancesManifest []Snapshot + InstanceSnapshots []Snapshot + + // List of files included + Manifest []string +} + +func (backup Backup) String() string { + var builder strings.Builder + backup.Report(&builder) + return builder.String() +} + +// Report writes a report from backup into w +func (backup Backup) Report(w io.Writer) { + // TODO: Errors + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + + io.WriteString(w, "======= Backup =======\n") + + fmt.Fprintf(w, "Start: %s\n", backup.StartTime) + fmt.Fprintf(w, "End: %s\n", backup.EndTime) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Description =======\n") + encoder.Encode(backup.Description) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Errors =======\n") + fmt.Fprintf(w, "Panic: %v\n", backup.ErrPanic) + fmt.Fprintf(w, "SQLErr: %s\n", backup.SQLErr) + fmt.Fprintf(w, "TSErr: %s\n", backup.TSErr) + fmt.Fprintf(w, "ConfigFileErr: %s\n", backup.ConfigFileErr) + fmt.Fprintf(w, "InstanceListErr: %s\n", backup.InstanceListErr) + + io.WriteString(w, "\n") + + io.WriteString(w, "======= Config Files =======\n") + encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest + + io.WriteString(w, "======= Snapshots =======\n") + for _, s := range backup.InstanceSnapshots { + io.WriteString(w, s.String()) + io.WriteString(w, "\n") + } + + io.WriteString(w, "======= Manifest =======\n") + for _, file := range backup.Manifest { + io.WriteString(w, file+"\n") + } + + io.WriteString(w, "\n") } func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) { @@ -44,7 +107,15 @@ func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) backup.ErrPanic = recover() }() - backup.run(io, dis) + // do the create keeping track of time! + logging.LogOperation(func() error { + backup.StartTime = time.Now() + backup.run(io, dis) + backup.EndTime = time.Now() + + return nil + }, io, "Writing backup files") + return } @@ -53,7 +124,7 @@ var errBackupSkipFile = errors.New("") func (backup *Backup) run(io stream.IOStream, dis *Distillery) { // create a wait group, and message channel wg := &sync.WaitGroup{} - messages := make(chan string, 4) + files := make(chan string, 4) // backup the sql wg.Add(1) @@ -61,7 +132,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { defer wg.Done() sqlPath := filepath.Join(backup.Description.Dest, "sql.sql") - messages <- sqlPath + files <- sqlPath sql, err := os.Create(sqlPath) if err != nil { @@ -80,7 +151,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { defer wg.Done() tsPath := filepath.Join(backup.Description.Dest, "triplestore") - messages <- tsPath + files <- tsPath // directly store the result backup.TSErr = dis.Triplestore().BackupAll(tsPath) @@ -97,15 +168,15 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { return } - files := []string{ - filepath.Join(dis.Config.DeployRoot, ".env"), // TODO: put the name of the configuration file somewhere - filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable + configs := []string{ + dis.Config.ConfigPath, + filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable dis.Config.SelfOverridesFile, dis.Config.GlobalAuthorizedKeysFile, } - backup.ConfigFilesManifest = make(map[string]error, len(files)) - for _, src := range files { + backup.ConfigFilesManifest = make(map[string]error, len(configs)) + for _, src := range configs { if !fsx.IsFile(src) { backup.ConfigFilesManifest[src] = errBackupSkipFile continue @@ -113,7 +184,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { dest := filepath.Join(cfgBackupDir, filepath.Base(src)) // copy the config file and store the error message - messages <- src + files <- src backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src) } }() @@ -138,9 +209,9 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0) - backup.InstancesManifest = make([]Snapshot, len(instances)) + backup.InstanceSnapshots = make([]Snapshot, len(instances)) for i, instance := range instances { - backup.InstancesManifest[i] = func() Snapshot { + backup.InstanceSnapshots[i] = func() Snapshot { dir := filepath.Join(instancesBackupDir, instance.Slug) if err := os.Mkdir(dir, fs.ModeDir); err != nil { return Snapshot{ @@ -148,7 +219,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { } } - messages <- dir + files <- dir return instance.Snapshot(iochild, SnapshotDescription{ Dest: dir, }) @@ -160,13 +231,29 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { // wait for the group, then close the message channel. go func() { wg.Wait() - close(messages) + close(files) }() - // print out all the messages - for message := range messages { - io.Println(message) + for file := range files { + // get the relative path to the root of the manifest. + // nothing *should* go wrong, but in case it does, use the original path. + path, err := filepath.Rel(backup.Description.Dest, file) + if err != nil { + path = file + } + + // write it to the command line + // and also add it to the manifest + io.Printf("\033[2K\r%s", path) + backup.Manifest = append(backup.Manifest, path) } + slices.Sort(backup.Manifest) // backup the manifest + io.Println("") + + // sort the instances manifest + slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool { + return a.Instance.Slug < b.Instance.Slug + }) } // WriteReport writes out the report belonging to this backup. @@ -184,7 +271,7 @@ func (backup Backup) WriteReport(io stream.IOStream) error { defer report.Close() // print the report into it! - _, err = fmt.Fprintf(report, "%#v\n", backup) + _, err = report.WriteString(backup.String()) return err }, io, "Writing backup report") } diff --git a/internal/env/snapshot.go b/internal/env/snapshot.go index eeb0bb7..cb5ffc7 100644 --- a/internal/env/snapshot.go +++ b/internal/env/snapshot.go @@ -1,10 +1,13 @@ package env import ( + "encoding/json" "fmt" + "io" "io/fs" "os" "path/filepath" + "strings" "sync" "time" @@ -13,6 +16,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/password" "github.com/tkw1536/goprogram/stream" + "golang.org/x/exp/slices" ) // SnapshotsDir returns the path that contains all snapshot related data. @@ -79,17 +83,76 @@ type Snapshot struct { Description SnapshotDescription Instance bookkeeping.Instance - // various error states, which are ignored when creating the snapshot - ErrPanic interface{} // panic, if any + // Start and End Time of the snapshot + StartTime time.Time + EndTime time.Time + // Generic Panic that may have occured + ErrPanic interface{} + + // Errors during starting and stopping the system ErrStart error ErrStop error + // List of files included + Manifest []string + + // Errors during other parts ErrBookkeep error ErrPathbuilder error ErrFilesystem error ErrTriplestore error - ErrSSQL error + ErrSQL error +} + +func (snapshot Snapshot) String() string { + var builder strings.Builder + snapshot.Report(&builder) + return builder.String() +} + +// Report writes a report from snapshot into w +func (snapshot Snapshot) Report(w io.Writer) { + // TODO: Errors of the writer! + encoder := json.NewEncoder(w) + encoder.SetIndent("", " ") + + io.WriteString(w, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n") + + fmt.Fprintf(w, "Slug: %s\n", snapshot.Instance.Slug) + fmt.Fprintf(w, "Dest: %s\n", snapshot.Description.Dest) + + fmt.Fprintf(w, "Start: %s\n", snapshot.StartTime) + fmt.Fprintf(w, "End: %s\n", snapshot.EndTime) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Description =======\n") + encoder.Encode(snapshot.Description) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Instance =======\n") + encoder.Encode(snapshot.Instance) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Errors =======\n") + fmt.Fprintf(w, "Panic: %v\n", snapshot.ErrPanic) + fmt.Fprintf(w, "Start: %s\n", snapshot.ErrStart) + fmt.Fprintf(w, "Stop: %s\n", snapshot.ErrStop) + fmt.Fprintf(w, "Bookkeep: %s\n", snapshot.ErrBookkeep) + fmt.Fprintf(w, "Pathbuilder: %s\n", snapshot.ErrPathbuilder) + fmt.Fprintf(w, "Filesystem: %s\n", snapshot.ErrFilesystem) + fmt.Fprintf(w, "Triplestore: %s\n", snapshot.ErrTriplestore) + fmt.Fprintf(w, "SQL: %s\n", snapshot.ErrSQL) + io.WriteString(w, "\n") + + io.WriteString(w, "======= Manifest =======\n") + for _, file := range snapshot.Manifest { + io.WriteString(w, file+"\n") + } + + io.WriteString(w, "\n") + + io.WriteString(w, "======= End Snapshot Report "+snapshot.Instance.Slug+" =======\n") } // Snapshot creates a new snapshot of this instance into dest @@ -98,13 +161,19 @@ func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) snapshot.Description = desc snapshot.Instance = instance.Instance - // catch anything critical that happened during the snapshot + // capture anything critical, and write the end time defer func() { snapshot.ErrPanic = recover() }() - // and do the create! - snapshot.create(io, instance) + // do the create keeping track of time! + logging.LogOperation(func() error { + snapshot.StartTime = time.Now() + snapshot.create(io, instance) + snapshot.EndTime = time.Now() + + return nil + }, io, "Writing snapshot files") return } @@ -124,7 +193,7 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { // create a wait group, and message channel wg := &sync.WaitGroup{} - messages := make(chan string, 4) + files := make(chan string, 4) // write bookkeeping information wg.Add(1) @@ -132,7 +201,7 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { defer wg.Done() bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt") - messages <- bkPath + files <- bkPath info, err := os.Create(bkPath) if err != nil { @@ -147,12 +216,13 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { }() // write pathbuilders + // TODO: Move this outside of the up/down stuff! wg.Add(1) go func() { defer wg.Done() pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders") - messages <- pbPath + files <- pbPath // create the directory! if err := os.Mkdir(pbPath, fs.ModeDir); err != nil { @@ -170,14 +240,10 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { defer wg.Done() fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase)) - if err := os.Mkdir(fsPath, fs.ModeDir); err != nil { - snapshot.ErrFilesystem = err - return - } // copy over whatever is in the base directory snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { - messages <- dst + files <- dst }) }() @@ -187,11 +253,12 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { defer wg.Done() tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq") - messages <- tsPath + files <- tsPath nquads, err := os.Create(tsPath) if err != nil { snapshot.ErrTriplestore = err + return } defer nquads.Close() @@ -205,17 +272,17 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { defer wg.Done() sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql") - messages <- sqlPath + files <- sqlPath sql, err := os.Create(sqlPath) if err != nil { - snapshot.ErrSSQL = err + snapshot.ErrSQL = err return } defer sql.Close() // directly store the result - snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase) + snapshot.ErrSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase) }() // TODO: Backup the docker image @@ -223,13 +290,26 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { // wait for the group, then close the message channel. go func() { wg.Wait() - close(messages) + close(files) }() - // print out all the messages - for message := range messages { - io.Println(message) + for file := range files { + // get the relative path to the root of the manifest. + // nothing *should* go wrong, but in case it does, use the original path. + path, err := filepath.Rel(snapshot.Description.Dest, file) + if err != nil { + path = file + } + + // write it to the command line + // and also add it to the manifest + io.Printf("\033[2K\r%s", path) + snapshot.Manifest = append(snapshot.Manifest, path) } + io.Println("") + + // make sure the manifest is sorted! + slices.Sort(snapshot.Manifest) } // WriteReport writes out the report belonging to this snapshot. @@ -247,7 +327,7 @@ func (snapshot Snapshot) WriteReport(io stream.IOStream) error { defer report.Close() // print the report into it! - _, err = fmt.Fprintf(report, "%#v\n", snapshot) + _, err = report.WriteString(snapshot.String()) return err }, io, "Writing snapshot report") } diff --git a/pkg/fsx/copy.go b/pkg/fsx/copy.go index bad0d32..55e6045 100644 --- a/pkg/fsx/copy.go +++ b/pkg/fsx/copy.go @@ -3,13 +3,16 @@ package fsx import ( "errors" "io" + "io/fs" "os" "path/filepath" ) -var ErrCopySameFile = errors.New("src and dst must be different files") +var ErrCopySameFile = errors.New("src and dst must be different") // CopyFile copies a file from src to dst. +// When src points to a symbolic link, will copy the symbolic link. +// // When dst and src are the same file, returns ErrCopySameFile. func CopyFile(dst, src string) error { if SameFile(src, dst) { @@ -41,10 +44,37 @@ func CopyFile(dst, src string) error { return err } -var ErrCopyNoDirectory = errors.New("dst is not a directory") +// CopyLink copies a link from src to dst. +// If dst already exists, it is deleted and then re-created. +func CopyLink(dst, src string) error { + // if they're the same file that is an error + if SameFile(dst, src) { + return ErrCopySameFile + } + + // read the link target + target, err := os.Readlink(src) + if err != nil { + return err + } + + // delete it if it already exists + if Exists(dst) { + if err := os.Remove(dst); err != nil { + return err + } + } + + // make the symbolic link! + return os.Symlink(target, dst) +} + +var ErrDstFile = errors.New("dst is a file") // CopyDirectory copies the directory src to dst recursively. -// The destination directory must exist, or an error is returned. +// +// Existing files and directories are overwritten. +// When a directory already exists, additional files are not deleted. // // onCopy, when not nil, is called for each file or directory being copied. func CopyDirectory(dst, src string, onCopy func(dst, src string)) error { @@ -52,56 +82,51 @@ func CopyDirectory(dst, src string, onCopy func(dst, src string)) error { if SameFile(src, dst) { return ErrCopySameFile } - if !IsDirectory(dst) { - return ErrCopyNoDirectory + if IsFile(dst) { + return ErrDstFile } - // call onCopy for this directory! - if onCopy != nil { - onCopy(dst, src) - } - - // iterate over the entries or bail out - entries, err := os.ReadDir(src) - if err != nil { - return err - } - - for _, entry := range entries { - name := entry.Name() - eDest := filepath.Join(dst, name) - eSrc := filepath.Join(src, name) - - // it is not a directory => Use CopyFile - if !entry.IsDir() { - if onCopy != nil { - onCopy(eDest, eSrc) - } - - // do the copy! - if err := CopyFile(eDest, eSrc); err != nil { - return err - } - - continue - } - - // find out the mode of the entry - eInfo, err := entry.Info() + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } - // make the target directory - if err := os.Mkdir(eDest, eInfo.Mode()); err != nil { + // determine the real target path + var relpath string + relpath, err = filepath.Rel(src, path) + if err != nil { + return err + } + dst := filepath.Join(dst, relpath) + + // call the hook + if onCopy != nil { + onCopy(dst, src) + } + + // stat the directory, so that we can get mode, and info later! + info, err := d.Info() + if err != nil { return err } - // do the copy! - if err := CopyDirectory(eDest, eSrc, onCopy); err != nil { - return err + // if we have a symbolic link, copy the link! + if info.Mode()&os.ModeSymlink != 0 { + return CopyLink(dst, path) } - } - return nil + // if we got a file, we should copy it normally + if !d.IsDir() { + return CopyFile(dst, path) + } + + // create the directory, but ignore an error if the directory already exists. + // this is so that we can copy one tree into another tree. + err = os.Mkdir(dst, info.Mode()) + if os.IsExist(err) && IsDirectory(dst) { + err = nil + } + + return err + }) } diff --git a/pkg/fsx/type.go b/pkg/fsx/type.go index 74c8993..64c4e5e 100644 --- a/pkg/fsx/type.go +++ b/pkg/fsx/type.go @@ -7,7 +7,7 @@ import ( // Exists checks if the given path exists func Exists(path string) bool { - _, err := os.Stat(path) + _, err := os.Lstat(path) return err == nil } @@ -22,3 +22,9 @@ func IsFile(path string) bool { info, err := os.Stat(path) return err == nil && info.Mode().IsRegular() } + +// IsLink checks if the provided path exists and is a symlink +func IsLink(path string) bool { + info, err := os.Lstat(path) + return err == nil && info.Mode()&os.ModeSymlink != 0 +} diff --git a/pkg/targz/targz.go b/pkg/targz/targz.go index 7aa0c9f..b147e2a 100644 --- a/pkg/targz/targz.go +++ b/pkg/targz/targz.go @@ -31,7 +31,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64, defer tarHandle.Close() // and walk through it! - err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { + err = filepath.WalkDir(src, func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } @@ -43,10 +43,17 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64, return err } + // call the oncopy! if onCopy != nil { onCopy(relpath, path) } + // read mode etc + info, err := entry.Info() + if err != nil { + return err + } + // create a file info header! tInfo, err := tar.FileInfoHeader(info, relpath) if err != nil { @@ -60,7 +67,7 @@ func Package(dst, src string, onCopy func(rel string, src string)) (count int64, } // a directory => no more writing required - if info.IsDir() { + if entry.IsDir() { return nil }