From d818cb93a513428da180c445e2256b035093b7bc Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Wed, 7 Sep 2022 11:02:50 +0200 Subject: [PATCH] Rename backups => snapshot --- cmd/backup.go | 106 -------------------------------- cmd/snapshot.go | 110 ++++++++++++++++++++++++++++++++++ cmd/system_update.go | 4 +- cmd/wdcli/main.go | 2 +- env/backup.go | 52 ---------------- env/snapshot.go | 50 ++++++++++++++++ internal/fsx/type.go | 5 ++ internal/password/password.go | 15 +++-- 8 files changed, 178 insertions(+), 166 deletions(-) delete mode 100644 cmd/backup.go create mode 100644 cmd/snapshot.go delete mode 100644 env/backup.go diff --git a/cmd/backup.go b/cmd/backup.go deleted file mode 100644 index 0de4284..0000000 --- a/cmd/backup.go +++ /dev/null @@ -1,106 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "path/filepath" - - wisski_distillery "github.com/FAU-CDI/wisski-distillery" - "github.com/FAU-CDI/wisski-distillery/env" - "github.com/FAU-CDI/wisski-distillery/internal/logging" - "github.com/FAU-CDI/wisski-distillery/internal/targz" - "github.com/tkw1536/goprogram/exit" -) - -// BackupInstance is the 'backup_instance' command -var BackupInstance wisski_distillery.Command = backupInstance{} - -type backupInstance struct { - Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"` - Positionals struct { - Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show info about"` - Outfile string `positional-arg-name:"OUTFILE" description:"Destination file to write backup to"` - } `positional-args:"true"` -} - -func (backupInstance) Description() wisski_distillery.Description { - return wisski_distillery.Description{ - Requirements: env.Requirements{ - NeedsConfig: true, - }, - Command: "backup_instance", - Description: "Makes a backup of a specific instance", - } -} - -var errBackupFailed = exit.Error{ - Message: "Failed to make a backup", - ExitCode: exit.ExitGeneric, -} - -func (bi backupInstance) Run(context wisski_distillery.Context) error { - dis := context.Environment - instance, err := dis.Instance(bi.Positionals.Slug) - if err != nil { - return err - } - - // TODO: Allow skipping backups of individual parts and make them concurrent! - - // start the backup and shutdown the instance (if requested) - logging.LogMessage(context.IOStream, "Creating backup of instance %s", bi.Positionals.Slug) - - // create a new temporary directory - logging.LogMessage(context.IOStream, "Creating temporary backup directory") - path, err := dis.NewInprogressBackupPath(instance.Slug) - if err != nil { - return errBackupFailed.Wrap(err) - } - defer func() { - logging.LogMessage(context.IOStream, "Removing temporary backup directory") - os.RemoveAll(path) // TODO: Turn this on again - }() - - // make a snapshot and write out the report also! - logging.LogOperation(func() error { - sreport := instance.Snapshot(context.IOStream, bi.Keepalive, path) - - logging.LogOperation(func() error { - reportPath := filepath.Join(path, "report.txt") - context.Println(reportPath) - - // create the report file! - report, err := os.Create(reportPath) - if err != nil { - return err - } - defer report.Close() - - // print the report into it! - _, err = fmt.Fprintf(report, "%#v\n", sreport) - return err - }, context.IOStream, "Writing snapshot report") - - return nil - }, context.IOStream, "Creating snapshot") - - // copy everything into the final file! - finalPath := bi.Positionals.Outfile - if finalPath == "" { - finalPath = dis.FinalBackupArchive(instance.Slug) - } - - if err := logging.LogOperation(func() error { - context.IOStream.Println(finalPath) - - targz.Package(finalPath, path, func(src string) { - context.Println(src) - }) - return err - }, context.IOStream, "Writing final backup"); err != nil { - return errBackupFailed.Wrap(err) - } - context.Printf("Wrote %s\n", finalPath) - - return nil -} diff --git a/cmd/snapshot.go b/cmd/snapshot.go new file mode 100644 index 0000000..e160f71 --- /dev/null +++ b/cmd/snapshot.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/targz" + "github.com/tkw1536/goprogram/exit" +) + +// Snapshot creates a snapshot of an instance +var Snapshot wisski_distillery.Command = snapshot{} + +type snapshot struct { + Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"` + + Positionals struct { + Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"` + Dest string `positional-arg-name:"DEST" description:"Destination path to write snapshot archive to. Defaults to the snapshots/archives/ directory"` + } `positional-args:"true"` +} + +func (snapshot) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "snapshot", + Description: "Generates a snapshot archive for the provided archive", + } +} + +var errSnapshotFailed = exit.Error{ + Message: "Failed to make a snapshot", + ExitCode: exit.ExitGeneric, +} + +func (bi snapshot) Run(context wisski_distillery.Context) error { + dis := context.Environment + instance, err := dis.Instance(bi.Positionals.Slug) + if err != nil { + return err + } + + // TODO: Allow skipping backups of individual parts and make them concurrent! + + // start the snapshot and shutdown the instance (if requested) + logging.LogMessage(context.IOStream, "Creating snapshot of instance %s", bi.Positionals.Slug) + + // create a new temporary directory + logging.LogMessage(context.IOStream, "Creating new snapshot staging directory") + sPath, err := dis.NewSnapshotStagingDir(instance.Slug) + if err != nil { + return errSnapshotFailed.Wrap(err) + } + defer func() { + logging.LogMessage(context.IOStream, "Removing snapshot staging directory") + os.RemoveAll(sPath) + }() + + // take a snapshot into the staging area! + sreport := instance.Snapshot(context.IOStream, bi.Keepalive, sPath) + + // write out the report! + logging.LogOperation(func() error { + + logging.LogOperation(func() error { + reportPath := filepath.Join(sPath, "report.txt") + context.Println(reportPath) + + // create the report file! + report, err := os.Create(reportPath) + if err != nil { + return err + } + defer report.Close() + + // print the report into it! + _, err = fmt.Fprintf(report, "%#v\n", sreport) + return err + }, context.IOStream, "Writing snapshot report") + + return nil + }, context.IOStream, "Creating snapshot") + + // copy everything into the final archive + + finalPath := bi.Positionals.Dest + if finalPath == "" { + finalPath = dis.NewSnapshotArchivePath(instance.Slug) + } + + if err := logging.LogOperation(func() error { + context.IOStream.Println(finalPath) + + targz.Package(finalPath, sPath, func(src string) { + context.Println(src) + }) + return err + }, context.IOStream, "Writing final backup"); err != nil { + return errSnapshotFailed.Wrap(err) + } + context.Printf("Wrote %s\n", finalPath) + + return nil +} diff --git a/cmd/system_update.go b/cmd/system_update.go index 5133324..8a89bc1 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -81,8 +81,8 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { for _, d := range []string{ dis.Config.DeployRoot, dis.InstancesDir(), - dis.InprogressBackupPath(), - dis.FinalBackupPath(), + dis.SnapshotsStagingPath(), + dis.SnapshotsArchivePath(), } { context.Println(d) if err := os.MkdirAll(d, os.ModeDir); err != nil { diff --git a/cmd/wdcli/main.go b/cmd/wdcli/main.go index adf8e51..0847c11 100644 --- a/cmd/wdcli/main.go +++ b/cmd/wdcli/main.go @@ -44,7 +44,7 @@ func init() { wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration // backup & cron - wdcli.Register(cmd.BackupInstance) + wdcli.Register(cmd.Snapshot) // wdcli.Register(cmd.BackupAll) wdcli.Register(cmd.Cron) } diff --git a/env/backup.go b/env/backup.go deleted file mode 100644 index a1f9690..0000000 --- a/env/backup.go +++ /dev/null @@ -1,52 +0,0 @@ -package env - -import ( - "fmt" - "os" - "path/filepath" - "sync/atomic" - "time" -) - -func (dis Distillery) BackupDir() string { - return filepath.Join(dis.Config.DeployRoot, "backups") -} - -func (dis Distillery) InprogressBackupPath() string { - return filepath.Join(dis.BackupDir(), "inprogress") -} - -func (dis Distillery) FinalBackupPath() string { - return filepath.Join(dis.BackupDir(), "final") -} - -// NewFinalBackupFile returns the path to a new final backup file. -func (dis Distillery) FinalBackupArchive(prefix string) string { - counter := atomic.AddUint64(&globalBackupCounter, 1) - - // generate a new name with the current time, a global counter, and the prefix - name := fmt.Sprintf("%s-%d-%d.tar.gz", prefix, time.Now().Unix(), counter) - path := filepath.Join(dis.FinalBackupPath(), name) - - return path -} - -var globalBackupCounter uint64 - -// NewInprogressBackupPath returns the path to a new inprogress backup directory. -// The directory is guaranteed to have been freshly created. -func (dis Distillery) NewInprogressBackupPath(prefix string) (string, error) { - counter := atomic.AddUint64(&globalBackupCounter, 1) - - // generate a new name with the current time, a global counter, and the prefix - name := fmt.Sprintf("%s-%d-%d", prefix, time.Now().Unix(), counter) - path := filepath.Join(dis.InprogressBackupPath(), name) - - // create the directory - if err := os.Mkdir(path, os.ModeDir); err != nil { - return "", err - } - - // and it is here! - return path, nil -} diff --git a/env/snapshot.go b/env/snapshot.go index 1cee645..d1e1d99 100644 --- a/env/snapshot.go +++ b/env/snapshot.go @@ -6,12 +6,62 @@ import ( "os" "path/filepath" "sync" + "time" "github.com/FAU-CDI/wisski-distillery/internal/fsx" "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/password" "github.com/tkw1536/goprogram/stream" ) +// SnapshotsDir returns the path that contains all snapshot related data. +func (dis Distillery) SnapshotsDir() string { + return filepath.Join(dis.Config.DeployRoot, "snapshots") +} + +// SnapshotsStagingPath returns the path to the directory containing a temporary staging area for snapshots. +// Use NewSnapshotStagingDir to generate a new staging area. +func (dis Distillery) SnapshotsStagingPath() string { + return filepath.Join(dis.SnapshotsDir(), "staging") +} + +// SnapshotsArchivePath returns the path to the directory containing all exported archives. +// Use NewSnapshotArchivePath to generate a path to a new archive in this directory. +func (dis Distillery) SnapshotsArchivePath() string { + return filepath.Join(dis.SnapshotsDir(), "archives") +} + +// NewSnapshotArchivePath returns the path to a new archive with the provided prefix. +// The path is guaranteed to not exist. +func (dis Distillery) NewSnapshotArchivePath(prefix string) (path string) { + // TODO: Consider moving these into a subdirectory with the provided prefix. + for path == "" || fsx.Exists(path) { + name := dis.newSnapshotName(prefix) + ".tar.gz" + path = filepath.Join(dis.SnapshotsArchivePath(), name) + } + return +} + +// newSnapshot name returns a new basename for a snapshot with the provided prefix. +// The name is guaranteed to be unique within this process. +func (Distillery) newSnapshotName(prefix string) string { + suffix, _ := password.Password(64) // silently ignore any errors! + return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix) +} + +// NewSnapshotStagingDir returns the path to a new snapshot directory. +// The directory is guaranteed to have been freshly created. +func (dis Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) { + for path == "" || os.IsExist(err) { + path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix)) + err = os.Mkdir(path, os.ModeDir) + } + if err != nil { + path = "" + } + return +} + type SnapshotReport struct { Keepalive bool // was the instance alive while running the snapshot? Panic interface{} // was there a panic? diff --git a/internal/fsx/type.go b/internal/fsx/type.go index d4c7c72..c26f31e 100644 --- a/internal/fsx/type.go +++ b/internal/fsx/type.go @@ -2,6 +2,11 @@ package fsx import "os" +func Exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + func IsDirectory(path string) bool { info, err := os.Stat(path) return err == nil && info.Mode().IsDir() diff --git a/internal/password/password.go b/internal/password/password.go index 5d2f67c..06ec8b5 100644 --- a/internal/password/password.go +++ b/internal/password/password.go @@ -10,28 +10,33 @@ import ( // NOTE(twiesing): A bunch of scripts cannot properly handle the extra characters in the password. // For now it is disabled, but it should be re-enabled later. const PasswordCharSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // + "!@#$%&*" -const PasswordCharCount = len(PasswordCharSet) +var passwordCharCount = big.NewInt(int64(len(PasswordCharSet))) -// Password returns a randomly generated password with the provided length. +// Password returns a randomly generated string with the provided length. +// It consists of alphanumeric characters only. +// +// When an error occurs, it is guaranteed to return "", err. // [rand.Reader] is used as the source of randomness. func Password(length int) (string, error) { if length < 0 { panic("length < 0") } + // create a buffer to write the string to! var password strings.Builder password.Grow(length) for i := 0; i < length; i++ { - // grab a random index! - index, err := rand.Int(rand.Reader, big.NewInt(int64(PasswordCharCount))) + // grab a random bIndex! + bIndex, err := rand.Int(rand.Reader, passwordCharCount) if err != nil { return "", err } // and use that index! - if err := password.WriteByte(PasswordCharSet[int(index.Int64())]); err != nil { + index := int(bIndex.Int64()) + if err := password.WriteByte(PasswordCharSet[index]); err != nil { return "", err } }