From ea714aba8695e68b49cc8158f43737c8550a0776 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Mon, 3 Oct 2022 13:37:12 +0200 Subject: [PATCH] snapshots: Unify 'snapshot' and 'backup' code --- cmd/backup.go | 196 ++++++++++++++++--------- cmd/snapshot.go | 119 ++------------- internal/component/snapshots/report.go | 44 ------ 3 files changed, 142 insertions(+), 217 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 0088ca7..bfbec11 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -1,6 +1,9 @@ package cmd import ( + "io" + "path/filepath" + wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/component/snapshots" "github.com/FAU-CDI/wisski-distillery/internal/core" @@ -41,102 +44,155 @@ var errBackupFailed = exit.Error{ func (bk backupC) Run(context wisski_distillery.Context) error { dis := context.Environment - var err error + // prune old backups if !bk.NoPrune { defer logging.LogOperation(func() error { return dis.SnapshotManager().PruneBackups(context.IOStream) }, context.IOStream, "Pruning old backups") } - // determine the target path for the archive - var sPath string - if !bk.StagingOnly { - // regular mode: create a temporary staging directory - logging.LogMessage(context.IOStream, "Creating new backup staging directory") - sPath, err = dis.SnapshotManager().NewStagingDir("") - if err != nil { - return errSnapshotFailed.Wrap(err) - } - defer func() { - logging.LogMessage(context.IOStream, "Removing snapshot staging directory") - dis.Environment.RemoveAll(sPath) - }() + // do the handling + err := handleSnapshotLike(context, SnapshotFlags{ + Dest: bk.Positionals.Dest, + Slug: "", + Title: "Backup", + StagingOnly: bk.StagingOnly, + + Do: func(dest string) SnapshotLike { + backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{ + Dest: dest, + ConcurrentSnapshots: bk.ConcurrentSnapshots, + }) + return &backup + }, + }) + + if err != nil { + return errBackupFailed.Wrap(err) + } + return nil +} + +type SnapshotFlags struct { + Dest string + Slug string + Title string // "Backup" or "Snapshot" + StagingOnly bool + + Do func(dest string) SnapshotLike +} + +type SnapshotLike interface { + LogEntry() models.Snapshot + Report(w io.Writer) (int, error) +} + +func handleSnapshotLike(context wisski_distillery.Context, flags SnapshotFlags) (err error) { + dis := context.Environment + + // determine target paths + logging.LogMessage(context.IOStream, "Determining target paths") + var stagingDir, archivePath string + if flags.StagingOnly { + stagingDir = flags.Dest } else { - // staging mode: use dest as a destination - sPath = bk.Positionals.Dest - if sPath == "" { - sPath, err = dis.SnapshotManager().NewStagingDir("") - if err != nil { - return errSnapshotFailed.Wrap(err) - } - } - - // create the directory (if it doesn't already exist) - logging.LogMessage(context.IOStream, "Creating staging directory") - err = dis.Core.Environment.Mkdir(sPath, environment.DefaultDirPerm) - if !environment.IsExist(err) && err != nil { - return errSnapshotFailed.WithMessageF(err) - } - err = nil + archivePath = flags.Dest } - context.Println(sPath) + if stagingDir == "" { + stagingDir, err = dis.SnapshotManager().NewStagingDir(flags.Slug) + if err != nil { + return err + } + } + if !flags.StagingOnly && archivePath == "" { + archivePath = dis.SnapshotManager().NewArchivePath(flags.Slug) + } + context.Printf("Staging Directory: %s\n", stagingDir) + context.Printf("Archive Path: %s\n", archivePath) - var logEntry models.Snapshot + // create the staging directory + logging.LogMessage(context.IOStream, "Creating staging directory") + err = dis.Core.Environment.Mkdir(stagingDir, environment.DefaultDirPerm) + if !environment.IsExist(err) && err != nil { + return err + } + + // if it was requested to not do staging only + // we need the staging directory to be deleted at the end + if !flags.StagingOnly { + defer func() { + logging.LogMessage(context.IOStream, "Removing staging directory") + dis.Environment.RemoveAll(stagingDir) + }() + } + + // create the actual snapshot or backup + // write out the report + // and retain a log entry + var entry models.Snapshot logging.LogOperation(func() error { - backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{ - Dest: sPath, - ConcurrentSnapshots: bk.ConcurrentSnapshots, - }) - backup.WriteReport(dis.Core.Environment, context.IOStream) - logEntry = backup.LogEntry() - return nil - }, context.IOStream, "Generating Backup") + // do the snapshot! + sl := flags.Do(stagingDir) - // create the archive path - archivePath := bk.Positionals.Dest - if archivePath == "" { - archivePath = dis.SnapshotManager().NewArchivePath("") - } + // create a log entry + entry = sl.LogEntry() - // do the logging - if bk.Positionals.Dest == "" { - defer logging.LogOperation(func() error { - if bk.StagingOnly { - logEntry.Path = sPath - logEntry.Packed = false - } else { - logEntry.Path = archivePath - logEntry.Packed = true - } + // find the report path + reportPath := filepath.Join(stagingDir, "report.txt") + context.Println(reportPath) - return dis.Instances().AddSnapshotLog(logEntry) - }, context.IOStream, "Writing Log Entry") - } + // create the path + report, err := dis.Environment.Create(reportPath, environment.DefaultFilePerm) + if err != nil { + return err + } - // if we requested to only have a staging area, then we are done - if bk.StagingOnly { - context.Printf("Wrote %s\n", sPath) + // and write out the report + { + _, err := sl.Report(report) + return err + } + }, context.IOStream, "Generating %s", flags.Title) + + // if we only requested staging + // all that is left is to write the log entry + if flags.StagingOnly { + logging.LogMessage(context.IOStream, "Writing Log Entry") + + // write out the log entry + entry.Path = stagingDir + entry.Packed = false + dis.Instances().AddSnapshotLog(entry) + + context.Printf("Wrote %s\n", stagingDir) return nil } - // and write everything into it! - var count int64 + // package everything up as an archive! if err := logging.LogOperation(func() error { - context.IOStream.Println(archivePath) + var count int64 + defer func() { context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }() st := status.NewWithCompat(context.Stdout, 1) st.Start() defer st.Stop() - count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) { + count, err = targz.Package(dis.Core.Environment, archivePath, stagingDir, func(dst, src string) { st.Set(0, dst) }) - return err - }, context.IOStream, "Writing backup archive"); err != nil { - return errSnapshotFailed.Wrap(err) - } - context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) + return err + }, context.IOStream, "Writing archive"); err != nil { + return err + } + + // write out the log entry + logging.LogMessage(context.IOStream, "Writing Log Entry") + entry.Path = archivePath + entry.Packed = true + dis.Instances().AddSnapshotLog(entry) + + // and we're done! return nil } diff --git a/cmd/snapshot.go b/cmd/snapshot.go index ca061f5..7dcd692 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -1,17 +1,10 @@ package cmd import ( - "io/fs" - wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/component/snapshots" "github.com/FAU-CDI/wisski-distillery/internal/core" - "github.com/FAU-CDI/wisski-distillery/internal/models" - "github.com/FAU-CDI/wisski-distillery/pkg/environment" - "github.com/FAU-CDI/wisski-distillery/pkg/logging" - "github.com/FAU-CDI/wisski-distillery/pkg/targz" "github.com/tkw1536/goprogram/exit" - "github.com/tkw1536/goprogram/status" ) // Snapshot creates a snapshot of an instance @@ -43,111 +36,31 @@ var errSnapshotFailed = exit.Error{ } func (bi snapshot) Run(context wisski_distillery.Context) error { - // TODO: Cleanup this code! - dis := context.Environment + + // find the instance! instance, err := dis.Instances().WissKI(bi.Positionals.Slug) if err != nil { return err } - logging.LogMessage(context.IOStream, "Creating snapshot of instance %s", bi.Positionals.Slug) + // do a snapshot of it! + err = handleSnapshotLike(context, SnapshotFlags{ + Dest: bi.Positionals.Dest, + Slug: bi.Positionals.Slug, + Title: "Snapshot", + StagingOnly: bi.StagingOnly, - // determine the target path for the archive - var sPath string + Do: func(dest string) SnapshotLike { + snapshot := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{ + Dest: dest, + }) + return &snapshot + }, + }) - if !bi.StagingOnly { - // regular mode: create a temporary staging directory - logging.LogMessage(context.IOStream, "Creating new snapshot staging directory") - sPath, err = dis.SnapshotManager().NewStagingDir(instance.Slug) - if err != nil { - return errSnapshotFailed.Wrap(err) - } - defer func() { - logging.LogMessage(context.IOStream, "Removing snapshot staging directory") - dis.Core.Environment.RemoveAll(sPath) - }() - } else { - // staging mode: use dest as a destination - sPath = bi.Positionals.Dest - if sPath == "" { - sPath, err = dis.SnapshotManager().NewStagingDir(instance.Slug) - if err != nil { - return errSnapshotFailed.Wrap(err) - } - } - - // create the directory (if it doesn't already exist) - logging.LogMessage(context.IOStream, "Creating staging directory") - err = dis.Core.Environment.Mkdir(sPath, fs.ModePerm) - if !environment.IsExist(err) && err != nil { - return errSnapshotFailed.WithMessageF(err) - } - err = nil - } - context.Println(sPath) - - // TODO: Allow skipping backups of individual parts and make them concurrent! - - // take a snapshot into the staging area! - var logEntry models.Snapshot - logging.LogOperation(func() error { - sreport := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{ - Dest: sPath, - Keepalive: bi.Keepalive, - }) - - // write out the report, ignoring any errors! - sreport.WriteReport(dis.Core.Environment, context.IOStream) - - logEntry = sreport.LogEntry() - - return nil - }, context.IOStream, "Generating Snapshot") - - // create the archive path - archivePath := bi.Positionals.Dest - if archivePath == "" { - archivePath = dis.SnapshotManager().NewArchivePath("") - } - - // do the logging - if bi.Positionals.Dest == "" { - defer logging.LogOperation(func() error { - if bi.StagingOnly { - logEntry.Path = sPath - logEntry.Packed = false - } else { - logEntry.Path = archivePath - logEntry.Packed = true - } - - return dis.Instances().AddSnapshotLog(logEntry) - }, context.IOStream, "Writing Log Entry") - } - // if we requested to only have a staging area, then we are done - if bi.StagingOnly { - context.Printf("Wrote %s\n", sPath) - return nil - } - - // and write everything into it! - // TODO: Should we move the open call to here? - var count int64 - if err := logging.LogOperation(func() error { - context.IOStream.Println(archivePath) - - st := status.NewWithCompat(context.Stdout, 1) - st.Start() - defer st.Stop() - - count, err = targz.Package(dis.Core.Environment, archivePath, sPath, func(dst, src string) { - st.Set(0, dst) - }) - return err - }, context.IOStream, "Writing snapshot archive"); err != nil { + if err != nil { return errSnapshotFailed.Wrap(err) } - context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) return nil } diff --git a/internal/component/snapshots/report.go b/internal/component/snapshots/report.go index a72c34e..f48b607 100644 --- a/internal/component/snapshots/report.go +++ b/internal/component/snapshots/report.go @@ -4,13 +4,9 @@ import ( "encoding/json" "fmt" "io" - "path/filepath" "strings" "github.com/FAU-CDI/wisski-distillery/pkg/countwriter" - "github.com/FAU-CDI/wisski-distillery/pkg/environment" - "github.com/FAU-CDI/wisski-distillery/pkg/logging" - "github.com/tkw1536/goprogram/stream" ) func (snapshot Snapshot) String() string { @@ -65,26 +61,6 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) { return ww.Sum() } -// WriteReport writes out the report belonging to this snapshot. -// It is a separate function, to allow writing it indepenently of the rest. -func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error { - return logging.LogOperation(func() error { - reportPath := filepath.Join(snapshot.Description.Dest, "report.txt") - stream.Println(reportPath) - - // create the report file! - report, err := env.Create(reportPath, environment.DefaultFilePerm) - if err != nil { - return err - } - defer report.Close() - - // print the report into it! - _, err = io.WriteString(report, snapshot.String()) - return err - }, stream, "Writing snapshot report") -} - // Strings turns this backup into a string for the BackupReport. func (backup Backup) String() string { var builder strings.Builder @@ -132,23 +108,3 @@ func (backup Backup) Report(w io.Writer) (int, error) { return cw.Sum() } - -// WriteReport writes out the report belonging to this backup. -// It is a separate function, to allow writing it indepenently of the rest. -func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error { - return logging.LogOperation(func() error { - reportPath := filepath.Join(backup.Description.Dest, "report.txt") - stream.Println(reportPath) - - // create the report file! - report, err := env.Create(reportPath, environment.DefaultFilePerm) - if err != nil { - return err - } - defer report.Close() - - // print the report into it! - _, err = io.WriteString(report, backup.String()) - return err - }, stream, "Writing backup report") -}