diff --git a/TODO.md b/TODO.md index dfce516..9120b1c 100644 --- a/TODO.md +++ b/TODO.md @@ -28,28 +28,12 @@ Work in progress. - restructure resource files - Documentation - single malt +- snapshots: export xml from pathbuilder ## Migrating Individual Commands -- [ ] backup_all.sh -- [x] backup_instance.sh -- [x] blind_update.sh -- [x] blind_update_all.sh -- [x] cron_all.sh -- [x] info.sh -- [x] ls.sh -- [x] make_mysql_account.sh + - [ ] monday_full.sh - [ ] monday_short.sh -- [x] mysql.sh -- [x] provision.sh -- [x] purge.sh -- [x] rebuild.sh -- [x] rebuild_all.sh -- [x] reserve.sh -- [x] shell.sh -- [x] system_install.sh -- [x] system_update.sh -- [x] update_prefix_config.sh ## TO BE REMOVED - [ ] call_update_php_hack.sh diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000..44d164d --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "io/fs" + "os" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/targz" + "github.com/tkw1536/goprogram/exit" +) + +// Backup is the 'backup' command +var Backup wisski_distillery.Command = backup{} + +type backup struct { + StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"` + Positionals struct { + Dest string `positional-arg-name:"DEST" description:"Destination path to write backup archive to. Defaults to the snapshots/archives/ directory"` + } `positional-args:"true"` +} + +func (backup) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "backup", + Description: "Makes a backup of the entire distillery", + } +} + +var errBackupFailed = exit.Error{ + Message: "Failed to make a backup", + ExitCode: exit.ExitGeneric, +} + +func (bk backup) Run(context wisski_distillery.Context) error { + dis := context.Environment + var err error + + // determine the target path for the archive + var sPath string + if !bk.StagingOnly { + // regular mode: create a temporary staging directory + logging.LogMessage(context.IOStream, "Creating new snapshot staging directory") + sPath, err = dis.NewSnapshotStagingDir("") + if err != nil { + return errSnapshotFailed.Wrap(err) + } + defer func() { + logging.LogMessage(context.IOStream, "Removing snapshot staging directory") + os.RemoveAll(sPath) + }() + } else { + // staging mode: use dest as a destination + sPath = bk.Positionals.Dest + if sPath == "" { + sPath, err = dis.NewSnapshotStagingDir("") + if err != nil { + return errSnapshotFailed.Wrap(err) + } + } + + // create the directory (if it doesn't already exist) + logging.LogMessage(context.IOStream, "Creating staging directory") + err = os.Mkdir(sPath, fs.ModePerm) + if !os.IsExist(err) && err != nil { + return errSnapshotFailed.WithMessageF(err) + } + err = nil + } + context.Println(sPath) + + logging.LogOperation(func() error { + // take a snapshot into the staging area! + backup := dis.Backup(context.IOStream, env.BackupDescription{ + Dest: sPath, + }) + + // write out the report, ignoring any errors! + backup.WriteReport(context.IOStream) + + return nil + }, context.IOStream, "Generating Backup") + + // if we requested to only have a staging area, then we are done + if bk.StagingOnly { + context.Printf("Wrote %s\n", sPath) + return nil + } + + // create the archive path + archivePath := bk.Positionals.Dest + if archivePath == "" { + archivePath = dis.NewSnapshotArchivePath("") + } + + // and write everything into it! + // TODO: Should we move the open call to here? + var count int64 + if err := logging.LogOperation(func() error { + context.IOStream.Println(archivePath) + + count, err = targz.Package(archivePath, sPath, func(dst, src string) { + context.Println(dst) + }) + return err + }, context.IOStream, "Writing backup archive"); err != nil { + return errSnapshotFailed.Wrap(err) + } + context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) + + return nil +} diff --git a/cmd/snapshot.go b/cmd/snapshot.go index e160f71..63c0c3f 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -1,9 +1,8 @@ package cmd import ( - "fmt" + "io/fs" "os" - "path/filepath" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/env" @@ -16,7 +15,8 @@ import ( var Snapshot wisski_distillery.Command = snapshot{} type snapshot struct { - Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"` + Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"` + StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a snapshot archive, but only create a staging directory"` Positionals struct { Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"` @@ -46,65 +46,81 @@ func (bi snapshot) Run(context wisski_distillery.Context) error { return err } - // TODO: Allow skipping backups of individual parts and make them concurrent! - - // start the snapshot and shutdown the instance (if requested) logging.LogMessage(context.IOStream, "Creating snapshot of instance %s", bi.Positionals.Slug) - // create a new temporary directory - logging.LogMessage(context.IOStream, "Creating new snapshot staging directory") - sPath, err := dis.NewSnapshotStagingDir(instance.Slug) - if err != nil { - return errSnapshotFailed.Wrap(err) + // determine the target path for the archive + var sPath string + if !bi.StagingOnly { + // regular mode: create a temporary staging directory + logging.LogMessage(context.IOStream, "Creating new snapshot staging directory") + sPath, err = dis.NewSnapshotStagingDir(instance.Slug) + if err != nil { + return errSnapshotFailed.Wrap(err) + } + defer func() { + logging.LogMessage(context.IOStream, "Removing snapshot staging directory") + os.RemoveAll(sPath) + }() + } else { + // staging mode: use dest as a destination + sPath = bi.Positionals.Dest + if sPath == "" { + sPath, err = dis.NewSnapshotStagingDir(instance.Slug) + if err != nil { + return errSnapshotFailed.Wrap(err) + } + } + + // create the directory (if it doesn't already exist) + logging.LogMessage(context.IOStream, "Creating staging directory") + err = os.Mkdir(sPath, fs.ModePerm) + if !os.IsExist(err) && err != nil { + return errSnapshotFailed.WithMessageF(err) + } + err = nil } - defer func() { - logging.LogMessage(context.IOStream, "Removing snapshot staging directory") - os.RemoveAll(sPath) - }() + context.Println(sPath) + + // TODO: Allow skipping backups of individual parts and make them concurrent! // take a snapshot into the staging area! - sreport := instance.Snapshot(context.IOStream, bi.Keepalive, sPath) - - // write out the report! logging.LogOperation(func() error { + sreport := instance.Snapshot(context.IOStream, env.SnapshotDescription{ + Dest: sPath, + Keepalive: bi.Keepalive, + }) - logging.LogOperation(func() error { - reportPath := filepath.Join(sPath, "report.txt") - context.Println(reportPath) - - // create the report file! - report, err := os.Create(reportPath) - if err != nil { - return err - } - defer report.Close() - - // print the report into it! - _, err = fmt.Fprintf(report, "%#v\n", sreport) - return err - }, context.IOStream, "Writing snapshot report") + // write out the report, ignoring any errors! + sreport.WriteReport(context.IOStream) return nil - }, context.IOStream, "Creating snapshot") + }, context.IOStream, "Generating Snapshot") - // copy everything into the final archive - - finalPath := bi.Positionals.Dest - if finalPath == "" { - finalPath = dis.NewSnapshotArchivePath(instance.Slug) + // if we requested to only have a staging area, then we are done + if bi.StagingOnly { + context.Printf("Wrote %s\n", sPath) + return nil } - if err := logging.LogOperation(func() error { - context.IOStream.Println(finalPath) + // create the archive path + archivePath := bi.Positionals.Dest + if archivePath == "" { + archivePath = dis.NewSnapshotArchivePath(instance.Slug) + } - targz.Package(finalPath, sPath, func(src string) { - context.Println(src) + // and write everything into it! + // TODO: Should we move the open call to here? + var count int64 + if err := logging.LogOperation(func() error { + context.IOStream.Println(archivePath) + + count, err = targz.Package(archivePath, sPath, func(dst, src string) { + context.Println(dst) }) return err - }, context.IOStream, "Writing final backup"); err != nil { + }, context.IOStream, "Writing snapshot archive"); err != nil { return errSnapshotFailed.Wrap(err) } - context.Printf("Wrote %s\n", finalPath) - + context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) return nil } diff --git a/cmd/prefix.go b/cmd/update_prefix_config.go similarity index 100% rename from cmd/prefix.go rename to cmd/update_prefix_config.go diff --git a/cmd/wdcli/main.go b/cmd/wdcli/main.go index 0847c11..3800146 100644 --- a/cmd/wdcli/main.go +++ b/cmd/wdcli/main.go @@ -45,7 +45,7 @@ func init() { // backup & cron wdcli.Register(cmd.Snapshot) - // wdcli.Register(cmd.BackupAll) + wdcli.Register(cmd.Backup) wdcli.Register(cmd.Cron) } diff --git a/distillery/backup_all.sh b/distillery/backup_all.sh deleted file mode 100644 index c3ed912..0000000 --- a/distillery/backup_all.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" - - -log_info " => Starting backup process. This might take a while. " -wait_for_sql - -BACKUP_SLUG="$(date +%Y%m%dT%H%M%S)-$(randompw)" -BACKUP_INSTANCE_DIR="$DEPLOY_BACKUP_INPROGRESS_DIR/$BACKUP_SLUG" -BACKUP_FINAL_FILE="$DEPLOY_BACKUP_FINAL_DIR/$BACKUP_SLUG.tar.gz" - -BACKUP_SQL_FILE="$BACKUP_INSTANCE_DIR/backup.sql" - -BACKUP_TRIPLESTORE_DIR="$BACKUP_INSTANCE_DIR/triplestore" -BACKUP_TRIPLESTORE_SYSTEM="$BACKUP_TRIPLESTORE_DIR/system.nq" - -BACKUP_FILESYSTEM_DIR="$BACKUP_INSTANCE_DIR/instances" - -# create the backup directories -log_info " => Making '$BACKUP_INSTANCE_DIR'" -mkdir -p "$BACKUP_INSTANCE_DIR" -mkdir -p "$DEPLOY_BACKUP_FINAL_DIR" - -function backup_everything() { - # backup the configuration - log_info " => Backing up configuration" - cp "$CONFIG_FILE" "$BACKUP_INSTANCE_DIR/.env" || true - - # Backup sql (complete) - log_info " => Backing up the SQL database" - dockerized_mysqldump --all-databases > "$BACKUP_SQL_FILE" || true - - # Backup triplestore (complete) - log_info " => Backing up Triplestore System" - mkdir -p "$BACKUP_TRIPLESTORE_DIR" - curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/SYSTEM/statements?infer=false" > "$BACKUP_TRIPLESTORE_SYSTEM" || true - - # backup triplestore (individual) - for REPO in `grep -oP '(?<=#repositoryID> ")[^"]+' $BACKUP_TRIPLESTORE_SYSTEM`; do - log_info " => Backing up Triplestore Repository '$REPO'" - curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/$REPO/statements?infer=false" > "$BACKUP_TRIPLESTORE_DIR/repo_$REPO.nq" || true - done - - # backup all the instances - log_info "=> Backing up instances" - for slug in $(sql_bookkeep_list); do - log_info "=> /bin/bash '$DIR/backup_instance.sh' '$slug' '$BACKUP_INSTANCE_DIR/${slug}.tar.gz'" - /bin/bash "$DIR/backup_instance.sh" "$slug" "$BACKUP_INSTANCE_DIR/${slug}.tar.gz" 2>&1 | tee "$BACKUP_INSTANCE_DIR/${slug}.log" || true - done -} - -# do the entire backup -backup_everything 2>&1 | tee "$BACKUP_LOG_FILE.log" - -# Package the backup into a .tar.gz -log_info " => Packaging '$BACKUP_FINAL_FILE'" -pushd "$BACKUP_INSTANCE_DIR" > /dev/null -tar --totals --checkpoint=10000 -zcf "$BACKUP_FINAL_FILE" . -popd > /dev/null - -# Clean up the unpacked backup -log_info " => Cleaning up '$BACKUP_INSTANCE_DIR'" -rm -rf "$BACKUP_INSTANCE_DIR" - -log_info " => Removing backups older than $MAX_BACKUP_AGE days" -find "$DEPLOY_BACKUP_FINAL_DIR" -type f -mtime "+$MAX_BACKUP_AGE" -print -exec rm -f {} \; \ No newline at end of file diff --git a/env/backup.go b/env/backup.go new file mode 100644 index 0000000..14270b2 --- /dev/null +++ b/env/backup.go @@ -0,0 +1,189 @@ +package env + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/stream" +) + +// backupDescription is a description for a backup +type BackupDescription struct { + Dest string // destination path +} + +// Snapshot represents the result of generating a snapshot +type Backup struct { + Description BackupDescription + + // various error states, which are ignored when creating the snapshot + ErrPanic interface{} + + SQLErr error + TSErr error + + ConfigFileErr error + ConfigFilesManifest map[string]error + + InstanceListErr error + InstancesManifest []Snapshot +} + +func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) { + backup.Description = description + + // catch anything critical that happened during the snapshot + defer func() { + backup.ErrPanic = recover() + }() + + backup.run(io, dis) + return +} + +var errBackupSkipFile = errors.New("") + +func (backup *Backup) run(io stream.IOStream, dis *Distillery) { + // create a wait group, and message channel + wg := &sync.WaitGroup{} + messages := make(chan string, 4) + + // backup the sql + wg.Add(1) + go func() { + defer wg.Done() + + sqlPath := filepath.Join(backup.Description.Dest, "sql.sql") + messages <- sqlPath + + sql, err := os.Create(sqlPath) + if err != nil { + backup.SQLErr = err + return + } + defer sql.Close() + + // directly store the result + backup.SQLErr = dis.SQL().BackupAll(io, sql) + }() + + // backup the triplestore + wg.Add(1) + go func() { + defer wg.Done() + + tsPath := filepath.Join(backup.Description.Dest, "triplestore") + messages <- tsPath + + // directly store the result + backup.TSErr = dis.Triplestore().BackupAll(tsPath) + }() + + // backup configuration files + wg.Add(1) + go func() { + defer wg.Done() + + cfgBackupDir := filepath.Join(backup.Description.Dest, "config") + if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil { + backup.ConfigFileErr = err + return + } + + files := []string{ + filepath.Join(dis.Config.DeployRoot, ".env"), // TODO: put the name of the configuration file somewhere + filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable + dis.Config.SelfOverridesFile, + dis.Config.GlobalAuthorizedKeysFile, + } + + backup.ConfigFilesManifest = make(map[string]error, len(files)) + for _, src := range files { + if !fsx.IsFile(src) { + backup.ConfigFilesManifest[src] = errBackupSkipFile + continue + } + dest := filepath.Join(cfgBackupDir, filepath.Base(src)) + + // copy the config file and store the error message + messages <- src + backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src) + } + }() + + // backup instances + wg.Add(1) + go func() { + defer wg.Done() + + instancesBackupDir := filepath.Join(backup.Description.Dest, "instances") + if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil { + backup.InstanceListErr = err + return + } + + // list all instances + instances, err := dis.AllInstances() + if err != nil { + backup.InstanceListErr = err + return + } + + iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0) + + backup.InstancesManifest = make([]Snapshot, len(instances)) + for i, instance := range instances { + backup.InstancesManifest[i] = func() Snapshot { + dir := filepath.Join(instancesBackupDir, instance.Slug) + if err := os.Mkdir(dir, fs.ModeDir); err != nil { + return Snapshot{ + ErrPanic: err, + } + } + + messages <- dir + return instance.Snapshot(iochild, SnapshotDescription{ + Dest: dir, + }) + }() + } + + }() + + // wait for the group, then close the message channel. + go func() { + wg.Wait() + close(messages) + }() + + // print out all the messages + for message := range messages { + io.Println(message) + } +} + +// WriteReport writes out the report belonging to this backup. +// It is a separate function, to allow writing it indepenently of the rest. +func (backup Backup) WriteReport(io stream.IOStream) error { + return logging.LogOperation(func() error { + reportPath := filepath.Join(backup.Description.Dest, "report.txt") + io.Println(reportPath) + + // create the report file! + report, err := os.Create(reportPath) + if err != nil { + return err + } + defer report.Close() + + // print the report into it! + _, err = fmt.Fprintf(report, "%#v\n", backup) + return err + }, io, "Writing backup report") +} diff --git a/env/component_sql.go b/env/component_sql.go index 1372a63..d2fbe1a 100644 --- a/env/component_sql.go +++ b/env/component_sql.go @@ -103,7 +103,21 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code" func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error { io = stream.NewIOStream(dest, io.Stderr, nil, 0) - code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--database", database) + code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database) + if err != nil { + return err + } + if code != 0 { + return errSQLBackup + } + return nil +} + +// BackupAll makes a backup of all sql databases +func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error { + io = stream.NewIOStream(dest, io.Stderr, nil, 0) + + code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases") if err != nil { return err } diff --git a/env/component_triplestore.go b/env/component_triplestore.go index 8ba64e9..2680dcf 100644 --- a/env/component_triplestore.go +++ b/env/component_triplestore.go @@ -8,6 +8,7 @@ import ( "io/fs" "mime/multipart" "net/http" + "os" "path/filepath" "time" @@ -237,6 +238,62 @@ func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error) return io.Copy(dst, res.Body) } +type Repository struct { + ID string `json:"id"` + Title string `json:"title"` + URI string `json:"uri"` + Type string `json:"type"` + SesameType string `json:"sesameType"` + Location string `json:"location"` + Readable bool `json:"readable"` + Writable bool `json:"writable"` + Local bool `json:"local"` +} + +func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error) { + res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json") + if err != nil { + return nil, err + } + defer res.Body.Close() + + err = json.NewDecoder(res.Body).Decode(&repos) + return +} + +// TriplestoreBackup backs up every graphdb instance into dst +func (ts TriplestoreComponent) BackupAll(dst string) error { + // list all the repositories + repos, err := ts.listRepositories() + if err != nil { + return err + } + + // create the base directory + if err := os.Mkdir(dst, fs.ModeDir); err != nil { + return err + } + + // iterate over all the repositories + for _, repo := range repos { + if rErr := (func(repo Repository) error { + name := filepath.Join(dst, repo.ID+".nq") + + dest, err := os.Create(name) + if err != nil { + return err + } + defer dest.Close() + + _, err = ts.Backup(dest, repo.ID) + return err + }(repo)); err == nil && rErr != nil { + err = rErr + } + } + return err +} + var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK") func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error { diff --git a/env/snapshot.go b/env/snapshot.go index d1e1d99..18c4781 100644 --- a/env/snapshot.go +++ b/env/snapshot.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" "github.com/FAU-CDI/wisski-distillery/internal/fsx" "github.com/FAU-CDI/wisski-distillery/internal/logging" "github.com/FAU-CDI/wisski-distillery/internal/password" @@ -46,6 +47,11 @@ func (dis Distillery) NewSnapshotArchivePath(prefix string) (path string) { // The name is guaranteed to be unique within this process. func (Distillery) newSnapshotName(prefix string) string { suffix, _ := password.Password(64) // silently ignore any errors! + if prefix == "" { + prefix = "backup" + } else { + prefix = "snapshot-" + prefix + } return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix) } @@ -62,36 +68,56 @@ func (dis Distillery) NewSnapshotStagingDir(prefix string) (path string, err err return } -type SnapshotReport struct { - Keepalive bool // was the instance alive while running the snapshot? - Panic interface{} // was there a panic? +// SnapshotDescription is a description for a snapshot +type SnapshotDescription struct { + Dest string // destination path + Keepalive bool // should we keep the instance alive while making the snapshot? +} - // errors for the various components of the Snapshot - StopErr error - StartErr error - BookkeepingErr error - FilesystemErr error - TriplestoreErr error - SQLErr error +// Snapshot represents the result of generating a snapshot +type Snapshot struct { + Description SnapshotDescription + Instance bookkeeping.Instance + + // various error states, which are ignored when creating the snapshot + ErrPanic interface{} // panic, if any + + ErrStart error + ErrStop error + + ErrBookkeep error + ErrFilesystem error + ErrTriplestore error + ErrSSQL error } // Snapshot creates a new snapshot of this instance into dest -func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest string) (report SnapshotReport) { +func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) { + // setup the snapshot + snapshot.Description = desc + snapshot.Instance = instance.Instance + // catch anything critical that happened during the snapshot defer func() { - report.Panic = recover() + snapshot.ErrPanic = recover() }() + // and do the create! + snapshot.create(io, instance) + + return +} + +func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { stack := instance.Stack() // stop the instance (unless it was explicitly asked to not do so!) - report.Keepalive = keepalive - if !keepalive { + if !snapshot.Description.Keepalive { logging.LogMessage(io, "Stopping instance") - report.StopErr = stack.Down(io) + snapshot.ErrStop = stack.Down(io) defer func() { logging.LogMessage(io, "Starting instance") - report.StartErr = stack.Up(io) + snapshot.ErrStart = stack.Up(io) }() } @@ -104,19 +130,19 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin go func() { defer wg.Done() - bkPath := filepath.Join(dest, "bookkeeping.txt") + bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt") messages <- bkPath info, err := os.Create(bkPath) if err != nil { - report.BookkeepingErr = err + snapshot.ErrBookkeep = err return } defer info.Close() // print whatever is in the database // TODO: This should be sql code, maybe gorm can do that? - _, report.BookkeepingErr = fmt.Fprintf(info, "%#v\n", instance.Instance) + _, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance) }() // backup the filesystem @@ -124,14 +150,14 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin go func() { defer wg.Done() - fsPath := filepath.Join(dest, filepath.Base(instance.FilesystemBase)) + fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase)) if err := os.Mkdir(fsPath, fs.ModeDir); err != nil { - report.FilesystemErr = err + snapshot.ErrFilesystem = err return } // copy over whatever is in the base directory - report.FilesystemErr = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { + snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { messages <- dst }) }() @@ -141,17 +167,17 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin go func() { defer wg.Done() - tsPath := filepath.Join(dest, instance.GraphDBRepository+".nq") + tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq") messages <- tsPath nquads, err := os.Create(tsPath) if err != nil { - report.TriplestoreErr = err + snapshot.ErrTriplestore = err } defer nquads.Close() // directly store the result - _, report.TriplestoreErr = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository) + _, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository) }() // backup the sql database @@ -159,18 +185,18 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin go func() { defer wg.Done() - sqlPath := filepath.Join(dest, instance.SqlDatabase+".sql") + sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql") messages <- sqlPath sql, err := os.Create(sqlPath) if err != nil { - report.SQLErr = err + snapshot.ErrSSQL = err return } defer sql.Close() // directly store the result - report.SQLErr = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase) + snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase) }() // TODO: Backup the docker image @@ -185,6 +211,24 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin for message := range messages { io.Println(message) } - - return +} + +// WriteReport writes out the report belonging to this snapshot. +// It is a separate function, to allow writing it indepenently of the rest. +func (snapshot Snapshot) WriteReport(io stream.IOStream) error { + return logging.LogOperation(func() error { + reportPath := filepath.Join(snapshot.Description.Dest, "report.txt") + io.Println(reportPath) + + // create the report file! + report, err := os.Create(reportPath) + if err != nil { + return err + } + defer report.Close() + + // print the report into it! + _, err = fmt.Fprintf(report, "%#v\n", snapshot) + return err + }, io, "Writing snapshot report") } diff --git a/internal/targz/targz.go b/internal/targz/targz.go index d06130a..8e54603 100644 --- a/internal/targz/targz.go +++ b/internal/targz/targz.go @@ -11,11 +11,11 @@ import ( // Package packages the directory src into dst. // onCopy, when not nil, is called for each file being copied into the archive. -func Package(dst, src string, onCopy func(src string)) error { +func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) { // create the target archive archive, err := os.Create(dst) if err != nil { - return err + return 0, err } defer archive.Close() @@ -28,21 +28,28 @@ func Package(dst, src string, onCopy func(src string)) error { defer tarHandle.Close() // and walk through it! - return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { - if onCopy != nil { - onCopy(path) - } - + err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } + // determine the relative path + var relpath string + relpath, err = filepath.Rel(src, path) + if err != nil { + return err + } + + if onCopy != nil { + onCopy(relpath, path) + } + // create a file info header! - tInfo, err := tar.FileInfoHeader(info, path) + tInfo, err := tar.FileInfoHeader(info, relpath) if err != nil { return err } - tInfo.Name = filepath.ToSlash(path) + tInfo.Name = filepath.ToSlash(relpath) // write it! if err := tarHandle.WriteHeader(tInfo); err != nil { @@ -62,8 +69,9 @@ func Package(dst, src string, onCopy func(src string)) error { defer handle.Close() // and copy it into the archive - _, err = io.Copy(tarHandle, handle) + ccount, err := io.Copy(tarHandle, handle) + count += ccount return err }) - + return }