From 2a14d93d3c9fb20ec395734faf0bee3a8b6804cd Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Mon, 5 Sep 2022 14:43:50 +0200 Subject: [PATCH] Implement initial 'wdcli backup_instance' command This commit performs an initial implementation of the 'backup_instance' command. --- TODO.md | 5 +- cmd/backup.go | 184 ++++++++++++++++++++++++++++++++++ cmd/system_update.go | 2 + cmd/wdcli/main.go | 4 +- distillery/backup_instance.sh | 108 -------------------- env/backup.go | 52 ++++++++++ env/dirs.go | 23 ----- env/instances.go | 5 + env/runtime.go | 13 +++ env/stack.go | 2 + env/stack_sql.go | 17 ++++ env/stack_triplestore.go | 15 +++ go.mod | 2 +- go.sum | 2 + internal/fsx/copy.go | 68 +++++++++++++ internal/stack/stack.go | 1 + internal/targz/targz.go | 69 +++++++++++++ 17 files changed, 437 insertions(+), 135 deletions(-) create mode 100644 cmd/backup.go delete mode 100644 distillery/backup_instance.sh create mode 100644 env/backup.go delete mode 100644 env/dirs.go create mode 100644 env/runtime.go create mode 100644 internal/targz/targz.go diff --git a/TODO.md b/TODO.md index 34889bb..2d074b0 100644 --- a/TODO.md +++ b/TODO.md @@ -18,6 +18,9 @@ Work in progress. ## Future Work - Move `provision_entrypoint.sh` into go +- Clean up the distillery code, by moving to seperate structs per component +- Rename backups to 'snapshots' and make them restorable + - Snapshot the docker images being used also! - Avoid running `docker compose` executable and shift it to a library - Automatically bootstrap the docker container sql connection (use proper environment variables) - Make error handling consistent @@ -28,7 +31,7 @@ Work in progress. ## Migrating Individual Commands - [ ] backup_all.sh -- [ ] backup_instance.sh +- [x] backup_instance.sh - [x] blind_update.sh - [x] blind_update_all.sh - [x] cron_all.sh diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000..222b2fb --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,184 @@ +package cmd + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/targz" + "github.com/tkw1536/goprogram/exit" +) + +// BackupInstance is the 'backup_instance' command +var BackupInstance wisski_distillery.Command = backupInstance{} + +type backupInstance struct { + Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"` + Positionals struct { + Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show info about"` + Outfile string `positional-arg-name:"OUTFILE" description:"Destination file to write backup to"` + } `positional-args:"true"` +} + +func (backupInstance) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "backup_instance", + Description: "Makes a backup of a specific instance", + } +} + +var errBackupFailed = exit.Error{ + Message: "Failed to make a backup", + ExitCode: exit.ExitGeneric, +} + +func (bi backupInstance) Run(context wisski_distillery.Context) error { + dis := context.Environment + instance, err := dis.Instance(bi.Positionals.Slug) + if err != nil { + return err + } + + // TODO: Allow skipping backups of individual parts and make them concurrent! + + // start the backup and shutdown the instance (if requested) + logging.LogMessage(context.IOStream, "Creating backup of instance %s", bi.Positionals.Slug) + + // create a new temporary directory + logging.LogMessage(context.IOStream, "Creating temporary backup directory") + path, err := dis.NewInprogressBackupPath(instance.Slug) + if err != nil { + return errBackupFailed.Wrap(err) + } + defer func() { + logging.LogMessage(context.IOStream, "Removing temporary backup directory") + context.Println(path) + // os.RemoveAll(path) // TODO: Turn this on again + }() + + // make the snapshot! + // TODO: Ignore errors here, and write them into the snapshot instance + if err := bi.makeSnapshot(context, path, instance); err != nil { + return errBackupFailed.WithMessageF(err) + } + + // copy everything into the final file! + finalPath := bi.Positionals.Outfile + if finalPath == "" { + finalPath = dis.FinalBackupArchive(instance.Slug) + } + + if err := logging.LogOperation(func() error { + context.IOStream.Println(finalPath) + + targz.Package(finalPath, path, func(src string) { + context.Println(src) + }) + return err + }, context.IOStream, "Writing final backup"); err != nil { + return errBackupFailed.Wrap(err) + } + context.Printf("Wrote %s\n", finalPath) + + return nil +} + +// makeSnapshot makes a snapshot of the directory into the given directory! +// +// TODO: Return a SnapshotReport object, and only check what was actually copied +func (bi backupInstance) makeSnapshot(context wisski_distillery.Context, path string, instance env.Instance) error { + dis := context.Environment + stack := instance.Stack() + + if !bi.Keepalive { + logging.LogMessage(context.IOStream, "Stopping instance") + if err := stack.Down(context.IOStream); err != nil { + return err + } + defer func() { + logging.LogMessage(context.IOStream, "Starting instance") + stack.Up(context.IOStream) + }() + } + + // backup up bookkeeping info! + if err := logging.LogOperation(func() error { + bkPath := filepath.Join(path, "bookkeeping.txt") + context.IOStream.Println(bkPath) + + // create the backup file! + info, err := os.Create(bkPath) + if err != nil { + return err + } + defer info.Close() + + // print whatever is in the bookkeeping instance + _, err = fmt.Fprintf(info, "%#v\n", instance.Instance) + return err + }, context.IOStream, "Backing up Bookkeping Information"); err != nil { + return errBackupFailed.Wrap(err) + } + + // backup the filesystem! + if err := logging.LogOperation(func() error { + // create a backup directory + fsPath := filepath.Join(path, filepath.Base(instance.FilesystemBase)) + if err := os.Mkdir(fsPath, fs.ModeDir); err != nil { + return err + } + + return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) { + context.IOStream.Println(src) + }) + }, context.IOStream, "Backing up filesystem"); err != nil { + return errBackupFailed.Wrap(err) + } + + // backup the the triplestore! + if err := logging.LogOperation(func() error { + tsPath := filepath.Join(path, instance.GraphDBRepository+".nq") + context.IOStream.Println(tsPath) + + // create the backup file! + nquads, err := os.Create(tsPath) + if err != nil { + return err + } + defer nquads.Close() + + // TODO: Add a progress bar? + _, err = dis.TriplestoreBackup(nquads, instance.GraphDBRepository) + return err + }, context.IOStream, "Backing up Triplestore"); err != nil { + return errBackupFailed.Wrap(err) + } + + // backup the the sql database! + if err := logging.LogOperation(func() error { + sqlPath := filepath.Join(path, instance.SqlDatabase+".sql") + context.IOStream.Println(sqlPath) + + // create the backup file! + sql, err := os.Create(sqlPath) + if err != nil { + return err + } + defer sql.Close() + + // TODO: Add a progress bar? + return dis.SQLBackup(context.IOStream, sql, instance.SqlDatabase) + }, context.IOStream, "Backing up Triplestore"); err != nil { + return errBackupFailed.Wrap(err) + } + + return nil +} diff --git a/cmd/system_update.go b/cmd/system_update.go index 74cf883..a8c1c5e 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -160,6 +160,8 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { return errBootstrapTriplestore.WithMessageF(err) } + // TODO: Register cronjob in /etc/cron.d! + logging.LogMessage(context.IOStream, "System has been updated") return nil } diff --git a/cmd/wdcli/main.go b/cmd/wdcli/main.go index 609e4ec..adf8e51 100644 --- a/cmd/wdcli/main.go +++ b/cmd/wdcli/main.go @@ -41,12 +41,12 @@ func init() { // instance tasks wdcli.Register(cmd.Shell) wdcli.Register(cmd.BlindUpdate) - wdcli.Register(cmd.Cron) wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration // backup & cron - // wdcli.Register(cmd.BackupInstance) + wdcli.Register(cmd.BackupInstance) // wdcli.Register(cmd.BackupAll) + wdcli.Register(cmd.Cron) } // an error when no arguments are provided. diff --git a/distillery/backup_instance.sh b/distillery/backup_instance.sh deleted file mode 100644 index 731a708..0000000 --- a/distillery/backup_instance.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -unset DISABLE_LOG -require_slug_argument - -# if the site doesn't exist, I can't open a shell. -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I can't open a shell there. " - exit 1 -fi; - -# Read everything from the database -read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)" - -# prepare the backup -log_info " => Preparing Backup Configuration" - -BACKUP_SLUG="$SLUG-$(date +%Y%m%dT%H%M%S)-$(randompw)" -BACKUP_INSTANCE_DIR="$DEPLOY_BACKUP_INPROGRESS_DIR/$BACKUP_SLUG" - -BACKUP_LOG_FILE="$BACKUP_INSTANCE_DIR/backup.log" -BACKUP_INFO_FILE="$BACKUP_INSTANCE_DIR/info.txt" -BACKUP_SQL_FILE="$BACKUP_INSTANCE_DIR/$MYSQL_DATABASE.sql" -BACKUP_NQ_FILE="$BACKUP_INSTANCE_DIR/$GRAPHDB_REPO.nq" -BACKUP_FS_DIR="$BACKUP_INSTANCE_DIR/$SLUG" -mkdir -p "$BACKUP_FS_DIR" - -BACKUP_FINAL_FILE="$2" -if [ -z "$BACKUP_FINAL_FILE" ]; then - BACKUP_FINAL_FILE="$DEPLOY_BACKUP_FINAL_DIR/$BACKUP_SLUG.tar.gz" -fi - -echo "Destination: $BACKUP_FINAL_FILE" -if [ -z "$KEEPALIVE" ]; then - echo "Keepalive: false (set KEEPALIVE variable for a consistent state)" -else - echo "Keepalive: true (unset the KEEPALIVE variable for a consistent state)" -fi - -BACKUP_START="$(date +%s)" - -function do_the_backup() { - # cd into the right directory - cd "$INSTANCE_BASE_DIR" - - # stop - if [ -z "$KEEPALIVE" ]; then - log_info " => Shutting down running system" - docker-compose down - fi - - # system info - log_info " => Backup up system information" - /bin/bash "$DIR/info.sh" "$SLUG" > "$BACKUP_INFO_FILE" - - # database - log_info " => Backing up MySQL database '$MYSQL_DATABASE'" - dockerized_mysqldump --databases "$MYSQL_DATABASE" > "$BACKUP_SQL_FILE" || echo "Failed, continuing anyways ..." - - # triplestore - log_info " => Backing up GraphDB repository '$GRAPHDB_REPO'" - curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/${GRAPHDB_REPO}/statements?infer=false" > "$BACKUP_NQ_FILE" || echo "Failed, contiuing anyways ..." - - # filesystem - log_info " => Backing up filesystem from '$INSTANCE_BASE_DIR'" - cp -rpT "$INSTANCE_BASE_DIR" "$BACKUP_FS_DIR" || echo "Failed, continuing anyways ..." - - # restart - if [ -z "$KEEPALIVE" ]; then - log_info " => Starting up system" - docker-compose up -d - fi -} - -# do the actual backup, writing it to a file -do_the_backup 2>&1 | tee "$BACKUP_LOG_FILE" - -# list before packaging -log_info " => All backup files have been collected" -ls -alh "$BACKUP_INSTANCE_DIR" -du -hs "$BACKUP_INSTANCE_DIR" - -# package up the backup -log_info " => Packaging \"$BACKUP_FINAL_FILE\" " -pushd "$BACKUP_INSTANCE_DIR" > /dev/null -tar --totals --checkpoint=10000 -zcf "$BACKUP_FINAL_FILE" . -popd > /dev/null - -# delete the temporary directory -log_info " => Deleting temporary directories" -rm -r "$BACKUP_INSTANCE_DIR" - -# and finish! -log_info " => Finished making backup" - -DURATION=$[ $(date +%s) - ${BACKUP_START} ] -SIZE=$(wc -c < "$BACKUP_FINAL_FILE") - -echo "Output file: '$BACKUP_FINAL_FILE'" -echo "Size: $SIZE bytes" -echo "Duration: $DURATION seconds" \ No newline at end of file diff --git a/env/backup.go b/env/backup.go new file mode 100644 index 0000000..a1f9690 --- /dev/null +++ b/env/backup.go @@ -0,0 +1,52 @@ +package env + +import ( + "fmt" + "os" + "path/filepath" + "sync/atomic" + "time" +) + +func (dis Distillery) BackupDir() string { + return filepath.Join(dis.Config.DeployRoot, "backups") +} + +func (dis Distillery) InprogressBackupPath() string { + return filepath.Join(dis.BackupDir(), "inprogress") +} + +func (dis Distillery) FinalBackupPath() string { + return filepath.Join(dis.BackupDir(), "final") +} + +// NewFinalBackupFile returns the path to a new final backup file. +func (dis Distillery) FinalBackupArchive(prefix string) string { + counter := atomic.AddUint64(&globalBackupCounter, 1) + + // generate a new name with the current time, a global counter, and the prefix + name := fmt.Sprintf("%s-%d-%d.tar.gz", prefix, time.Now().Unix(), counter) + path := filepath.Join(dis.FinalBackupPath(), name) + + return path +} + +var globalBackupCounter uint64 + +// NewInprogressBackupPath returns the path to a new inprogress backup directory. +// The directory is guaranteed to have been freshly created. +func (dis Distillery) NewInprogressBackupPath(prefix string) (string, error) { + counter := atomic.AddUint64(&globalBackupCounter, 1) + + // generate a new name with the current time, a global counter, and the prefix + name := fmt.Sprintf("%s-%d-%d", prefix, time.Now().Unix(), counter) + path := filepath.Join(dis.InprogressBackupPath(), name) + + // create the directory + if err := os.Mkdir(path, os.ModeDir); err != nil { + return "", err + } + + // and it is here! + return path, nil +} diff --git a/env/dirs.go b/env/dirs.go deleted file mode 100644 index 1c7581e..0000000 --- a/env/dirs.go +++ /dev/null @@ -1,23 +0,0 @@ -package env - -import "path/filepath" - -func (dis Distillery) BackupDir() string { - return filepath.Join(dis.Config.DeployRoot, "backups") -} - -func (dis Distillery) RuntimeDir() string { - return filepath.Join(dis.Config.DeployRoot, "runtime") -} - -func (dis Distillery) RuntimeUtilsDir() string { - return filepath.Join(dis.Config.DeployRoot, "runtime", "utils") -} - -func (dis Distillery) InprogressBackupPath() string { - return filepath.Join(dis.BackupDir(), "inprogress") -} - -func (dis Distillery) FinalBackupPath() string { - return filepath.Join(dis.BackupDir(), "final") -} diff --git a/env/instances.go b/env/instances.go index bcd4c3b..5238644 100644 --- a/env/instances.go +++ b/env/instances.go @@ -30,6 +30,11 @@ var ErrInstanceNotFound = exit.Error{ ExitCode: exit.ExitGeneric, } +var errSQL = exit.Error{ + Message: "Unknown SQL Error %s", + ExitCode: exit.ExitGeneric, +} + // Instance returns the instance of the WissKI Distillery with the provided slug func (dis *Distillery) Instance(slug string) (i Instance, err error) { if err := dis.SQLWaitForConnection(); err != nil { diff --git a/env/runtime.go b/env/runtime.go new file mode 100644 index 0000000..31c2472 --- /dev/null +++ b/env/runtime.go @@ -0,0 +1,13 @@ +package env + +import "path/filepath" + +// RuntimeDir returns the path to the runtime directory +func (dis Distillery) RuntimeDir() string { + return filepath.Join(dis.Config.DeployRoot, "runtime") +} + +// RuntimeUtilsDir returns the path to the runtime utility dir +func (dis Distillery) RuntimeUtilsDir() string { + return filepath.Join(dis.Config.DeployRoot, "runtime", "utils") +} diff --git a/env/stack.go b/env/stack.go index 2a71ff1..abf5902 100644 --- a/env/stack.go +++ b/env/stack.go @@ -6,6 +6,8 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/stack" ) +// TODO: Move everything into specific subpackages + // Stacks returns the Stacks of this distillery func (dis *Distillery) Stacks() []stack.Installable { // TODO: Do we want to cache these stacks? diff --git a/env/stack_sql.go b/env/stack_sql.go index e17a907..2689f73 100644 --- a/env/stack_sql.go +++ b/env/stack_sql.go @@ -2,6 +2,7 @@ package env import ( "fmt" + "io" "io/fs" "time" @@ -76,6 +77,22 @@ func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) { return table, nil } +var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code") + +// SQLBackup makes a backup of the sql database into dest. +func (dis *Distillery) SQLBackup(io stream.IOStream, dest io.Writer, database string) error { + io = stream.NewIOStream(dest, io.Stderr, nil, 0) + + code, err := dis.SQLStack().Exec(io, "sql", "mysqldump", "--database", database) + if err != nil { + return err + } + if code != 0 { + return errSQLBackup + } + return nil +} + // SQLShell executes a mysql shell inside the SQLStack. func (dis *Distillery) SQLShell(io stream.IOStream, argv ...string) (int, error) { return dis.SQLStack().Exec(io, "sql", "mysql", argv...) diff --git a/env/stack_triplestore.go b/env/stack_triplestore.go index 85e4d58..105cf29 100644 --- a/env/stack_triplestore.go +++ b/env/stack_triplestore.go @@ -200,6 +200,21 @@ func (dis *Distillery) TriplestorePurgeRepo(repo string) error { return nil } +var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code") + +// TriplestoreBackup backs up the repository named repo into the writer dst. +func (dis *Distillery) TriplestoreBackup(dst io.Writer, repo string) (int64, error) { + res, err := dis.triplestoreRequest("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads") + if err != nil { + return 0, err + } + if res.StatusCode != http.StatusOK { + return 0, errTSBackupWrongStatusCode + } + defer res.Body.Close() + return io.Copy(dst, res.Body) +} + var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK") func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error { diff --git a/go.mod b/go.mod index dd2408e..67f73d9 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/alessio/shellescape v1.4.1 github.com/feiin/sqlstring v0.3.0 github.com/pkg/errors v0.9.1 - github.com/tkw1536/goprogram v0.0.9 + github.com/tkw1536/goprogram v0.0.10 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e gorm.io/driver/mysql v1.3.6 gorm.io/gorm v1.23.8 diff --git a/go.sum b/go.sum index 8fb8663..3248f41 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/tkw1536/goprogram v0.0.9 h1:y5bAWbiVRc47TjvpVDmyMtp5CgJXz1ultLOq+v9tfsA= github.com/tkw1536/goprogram v0.0.9/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4= +github.com/tkw1536/goprogram v0.0.10 h1:NRnAW46Vl9ro01eLDhlb7R56HsCls43h2anNXjwMPP4= +github.com/tkw1536/goprogram v0.0.10/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/fsx/copy.go b/internal/fsx/copy.go index 6fd49f2..a5e3c29 100644 --- a/internal/fsx/copy.go +++ b/internal/fsx/copy.go @@ -4,6 +4,7 @@ import ( "errors" "io" "os" + "path/filepath" ) var ErrCopySameFile = errors.New("src and dst must be different files") @@ -39,3 +40,70 @@ func CopyFile(dst, src string) error { _, err = io.Copy(dstFile, srcFile) return err } + +var ErrCopyNoDirectory = errors.New("dst is not a directory") + +// CopyDirectory copies the directory src to dst recursively. +// The destination directory must exist, or an error is returned. +// +// onCopy, when not nil, is called for each file or directory being copied. +func CopyDirectory(dst, src string, onCopy func(dst, src string)) error { + // TODO: Allow copying in parallel? Maybe with a mutex? + + // sanity checks + if src == dst { + return ErrCopySameFile + } + if !IsDirectory(dst) { + return ErrCopyNoDirectory + } + + // call onCopy for this directory! + if onCopy != nil { + onCopy(dst, src) + } + + // iterate over the entries or bail out + entries, err := os.ReadDir(src) + if err != nil { + return err + } + + for _, entry := range entries { + name := entry.Name() + eDest := filepath.Join(dst, name) + eSrc := filepath.Join(src, name) + + // it is not a directory => Use CopyFile + if !entry.IsDir() { + if onCopy != nil { + onCopy(eDest, eSrc) + } + + // do the copy! + if err := CopyFile(eDest, eSrc); err != nil { + return err + } + + continue + } + + // find out the mode of the entry + eInfo, err := entry.Info() + if err != nil { + return err + } + + // make the target directory + if err := os.Mkdir(eDest, eInfo.Mode()); err != nil { + return err + } + + // do the copy! + if err := CopyDirectory(eDest, eSrc, onCopy); err != nil { + return err + } + } + + return nil +} diff --git a/internal/stack/stack.go b/internal/stack/stack.go index 8a4b19f..4006442 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -76,6 +76,7 @@ func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...str if io.StdinIsATerminal() { compose = append(compose, "-ti") } + compose = append(compose, service) compose = append(compose, executable) compose = append(compose, args...) return ds.compose(io, compose...) diff --git a/internal/targz/targz.go b/internal/targz/targz.go new file mode 100644 index 0000000..d06130a --- /dev/null +++ b/internal/targz/targz.go @@ -0,0 +1,69 @@ +package targz + +import ( + "archive/tar" + "compress/gzip" + "io" + "io/fs" + "os" + "path/filepath" +) + +// Package packages the directory src into dst. +// onCopy, when not nil, is called for each file being copied into the archive. +func Package(dst, src string, onCopy func(src string)) error { + // create the target archive + archive, err := os.Create(dst) + if err != nil { + return err + } + defer archive.Close() + + // create a gzip writer + zipHandle := gzip.NewWriter(archive) + defer zipHandle.Close() + + // create a tar writer + tarHandle := tar.NewWriter(zipHandle) + defer tarHandle.Close() + + // and walk through it! + return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { + if onCopy != nil { + onCopy(path) + } + + if err != nil { + return err + } + + // create a file info header! + tInfo, err := tar.FileInfoHeader(info, path) + if err != nil { + return err + } + tInfo.Name = filepath.ToSlash(path) + + // write it! + if err := tarHandle.WriteHeader(tInfo); err != nil { + return err + } + + // a directory => no more writing required + if info.IsDir() { + return nil + } + + // open the file + handle, err := os.Open(path) + if err != nil { + return err + } + defer handle.Close() + + // and copy it into the archive + _, err = io.Copy(tarHandle, handle) + return err + }) + +}