diff --git a/TODO.md b/TODO.md index ce33208..dfce516 100644 --- a/TODO.md +++ b/TODO.md @@ -27,6 +27,7 @@ Work in progress. - Migrate the individual commands below - restructure resource files - Documentation +- single malt ## Migrating Individual Commands - [ ] backup_all.sh diff --git a/cmd/backup.go b/cmd/backup.go index b327b6c..0de4284 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -2,13 +2,11 @@ package cmd import ( "fmt" - "io/fs" "os" "path/filepath" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/env" - "github.com/FAU-CDI/wisski-distillery/internal/fsx" "github.com/FAU-CDI/wisski-distillery/internal/logging" "github.com/FAU-CDI/wisski-distillery/internal/targz" "github.com/tkw1536/goprogram/exit" @@ -63,11 +61,28 @@ func (bi backupInstance) Run(context wisski_distillery.Context) error { os.RemoveAll(path) // TODO: Turn this on again }() - // make the snapshot! - // TODO: Ignore errors here, and write them into the snapshot instance - if err := bi.makeSnapshot(context, path, instance); err != nil { - return errBackupFailed.WithMessageF(err) - } + // make a snapshot and write out the report also! + logging.LogOperation(func() error { + sreport := instance.Snapshot(context.IOStream, bi.Keepalive, path) + + logging.LogOperation(func() error { + reportPath := filepath.Join(path, "report.txt") + context.Println(reportPath) + + // create the report file! + report, err := os.Create(reportPath) + if err != nil { + return err + } + defer report.Close() + + // print the report into it! + _, err = fmt.Fprintf(report, "%#v\n", sreport) + return err + }, context.IOStream, "Writing snapshot report") + + return nil + }, context.IOStream, "Creating snapshot") // copy everything into the final file! finalPath := bi.Positionals.Outfile @@ -89,95 +104,3 @@ func (bi backupInstance) Run(context wisski_distillery.Context) error { return nil } - -// makeSnapshot makes a snapshot of the directory into the given directory! -// -// TODO: Return a SnapshotReport object, and only check what was actually copied -func (bi backupInstance) makeSnapshot(context wisski_distillery.Context, path string, instance env.Instance) error { - dis := context.Environment - stack := instance.Stack() - - if !bi.Keepalive { - logging.LogMessage(context.IOStream, "Stopping instance") - if err := stack.Down(context.IOStream); err != nil { - return err - } - defer func() { - logging.LogMessage(context.IOStream, "Starting instance") - stack.Up(context.IOStream) - }() - } - - // backup up bookkeeping info! - if err := logging.LogOperation(func() error { - bkPath := filepath.Join(path, "bookkeeping.txt") - context.IOStream.Println(bkPath) - - // create the backup file! - info, err := os.Create(bkPath) - if err != nil { - return err - } - defer info.Close() - - // print whatever is in the bookkeeping instance - _, err = fmt.Fprintf(info, "%#v\n", instance.Instance) - return err - }, context.IOStream, "Backing up Bookkeping Information"); err != nil { - return errBackupFailed.Wrap(err) - } - - // backup the filesystem! - if err := logging.LogOperation(func() error { - // create a backup directory - fsPath := filepath.Join(path, filepath.Base(instance.FilesystemBase)) - if err := os.Mkdir(fsPath, fs.ModeDir); err != nil { - return err - } - - return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { - context.IOStream.Println(src) - }) - }, context.IOStream, "Backing up filesystem"); err != nil { - return errBackupFailed.Wrap(err) - } - - // backup the the triplestore! - if err := logging.LogOperation(func() error { - tsPath := filepath.Join(path, instance.GraphDBRepository+".nq") - context.IOStream.Println(tsPath) - - // create the backup file! - nquads, err := os.Create(tsPath) - if err != nil { - return err - } - defer nquads.Close() - - // TODO: Add a progress bar? - _, err = dis.Triplestore().Backup(nquads, instance.GraphDBRepository) - return err - }, context.IOStream, "Backing up Triplestore"); err != nil { - return errBackupFailed.Wrap(err) - } - - // backup the the sql database! - if err := logging.LogOperation(func() error { - sqlPath := filepath.Join(path, instance.SqlDatabase+".sql") - context.IOStream.Println(sqlPath) - - // create the backup file! - sql, err := os.Create(sqlPath) - if err != nil { - return err - } - defer sql.Close() - - // TODO: Add a progress bar? - return dis.SQL().Backup(context.IOStream, sql, instance.SqlDatabase) - }, context.IOStream, "Backing up Triplestore"); err != nil { - return errBackupFailed.Wrap(err) - } - - return nil -} diff --git a/env/snapshot.go b/env/snapshot.go new file mode 100644 index 0000000..1cee645 --- /dev/null +++ b/env/snapshot.go @@ -0,0 +1,140 @@ +package env + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/tkw1536/goprogram/stream" +) + +type SnapshotReport struct { + Keepalive bool // was the instance alive while running the snapshot? + Panic interface{} // was there a panic? + + // errors for the various components of the Snapshot + StopErr error + StartErr error + BookkeepingErr error + FilesystemErr error + TriplestoreErr error + SQLErr error +} + +// Snapshot creates a new snapshot of this instance into dest +func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest string) (report SnapshotReport) { + // catch anything critical that happened during the snapshot + defer func() { + report.Panic = recover() + }() + + stack := instance.Stack() + + // stop the instance (unless it was explicitly asked to not do so!) + report.Keepalive = keepalive + if !keepalive { + logging.LogMessage(io, "Stopping instance") + report.StopErr = stack.Down(io) + defer func() { + logging.LogMessage(io, "Starting instance") + report.StartErr = stack.Up(io) + }() + } + + // create a wait group, and message channel + wg := &sync.WaitGroup{} + messages := make(chan string, 4) + + // write bookkeeping information + wg.Add(1) + go func() { + defer wg.Done() + + bkPath := filepath.Join(dest, "bookkeeping.txt") + messages <- bkPath + + info, err := os.Create(bkPath) + if err != nil { + report.BookkeepingErr = err + return + } + defer info.Close() + + // print whatever is in the database + // TODO: This should be sql code, maybe gorm can do that? + _, report.BookkeepingErr = fmt.Fprintf(info, "%#v\n", instance.Instance) + }() + + // backup the filesystem + wg.Add(1) + go func() { + defer wg.Done() + + fsPath := filepath.Join(dest, filepath.Base(instance.FilesystemBase)) + if err := os.Mkdir(fsPath, fs.ModeDir); err != nil { + report.FilesystemErr = err + return + } + + // copy over whatever is in the base directory + report.FilesystemErr = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { + messages <- dst + }) + }() + + // backup the graph db repository + wg.Add(1) + go func() { + defer wg.Done() + + tsPath := filepath.Join(dest, instance.GraphDBRepository+".nq") + messages <- tsPath + + nquads, err := os.Create(tsPath) + if err != nil { + report.TriplestoreErr = err + } + defer nquads.Close() + + // directly store the result + _, report.TriplestoreErr = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository) + }() + + // backup the sql database + wg.Add(1) + go func() { + defer wg.Done() + + sqlPath := filepath.Join(dest, instance.SqlDatabase+".sql") + messages <- sqlPath + + sql, err := os.Create(sqlPath) + if err != nil { + report.SQLErr = err + return + } + defer sql.Close() + + // directly store the result + report.SQLErr = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase) + }() + + // TODO: Backup the docker image + + // wait for the group, then close the message channel. + go func() { + wg.Wait() + close(messages) + }() + + // print out all the messages + for message := range messages { + io.Println(message) + } + + return +} diff --git a/go.sum b/go.sum index 3248f41..8a36b0d 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,6 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/tkw1536/goprogram v0.0.9 h1:y5bAWbiVRc47TjvpVDmyMtp5CgJXz1ultLOq+v9tfsA= -github.com/tkw1536/goprogram v0.0.9/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4= github.com/tkw1536/goprogram v0.0.10 h1:NRnAW46Vl9ro01eLDhlb7R56HsCls43h2anNXjwMPP4= github.com/tkw1536/goprogram v0.0.10/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= diff --git a/internal/stack/stack.go b/internal/stack/stack.go index 4006442..0c2a2b6 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -94,7 +94,7 @@ func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string if !io.StdinIsATerminal() { compose = append(compose, "-T") } - compose = append(compose, command) + compose = append(compose, service, command) compose = append(compose, args...) code, err := ds.compose(io, compose...)