rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wdcli: Implement backup & snapshot

Browse source 
This commit implements command backup and snapshot.
This commit is contained in:
Tom Wiesing 2022-09-07 16:01:09 +02:00
parent d818cb93a5
commit fc3b9170a6
No known key found for this signature in database
11 changed files with 535 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

20
TODO.md
View file

@ -28,28 +28,12 @@ Work in progress.
- restructure resource files
- Documentation
- single malt
- snapshots: export xml from pathbuilder
## Migrating Individual Commands
- [ ] backup_all.sh
- [x] backup_instance.sh
- [x] blind_update.sh
- [x] blind_update_all.sh
- [x] cron_all.sh
- [x] info.sh
- [x] ls.sh
- [x] make_mysql_account.sh
- [ ] monday_full.sh
- [ ] monday_short.sh
- [x] mysql.sh
- [x] provision.sh
- [x] purge.sh
- [x] rebuild.sh
- [x] rebuild_all.sh
- [x] reserve.sh
- [x] shell.sh
- [x] system_install.sh
- [x] system_update.sh
- [x] update_prefix_config.sh
## TO BE REMOVED
- [ ] call_update_php_hack.sh

116
cmd/backup.go Normal file
View file

@ -0,0 +1,116 @@
package cmd
import (
"io/fs"
"os"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/env"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/FAU-CDI/wisski-distillery/internal/targz"
"github.com/tkw1536/goprogram/exit"
)
// Backup is the 'backup' command
var Backup wisski_distillery.Command = backup{}
type backup struct {
StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"`
Positionals struct {
Dest string `positional-arg-name:"DEST" description:"Destination path to write backup archive to. Defaults to the snapshots/archives/ directory"`
} `positional-args:"true"`
}
func (backup) Description() wisski_distillery.Description {
return wisski_distillery.Description{
Requirements: env.Requirements{
NeedsConfig: true,
},
Command: "backup",
Description: "Makes a backup of the entire distillery",
}
}
var errBackupFailed = exit.Error{
Message: "Failed to make a backup",
ExitCode: exit.ExitGeneric,
}
func (bk backup) Run(context wisski_distillery.Context) error {
dis := context.Environment
var err error
// determine the target path for the archive
var sPath string
if !bk.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
sPath, err = dis.NewSnapshotStagingDir("")
if err != nil {
return errSnapshotFailed.Wrap(err)
}
defer func() {
logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
os.RemoveAll(sPath)
}()
} else {
// staging mode: use dest as a destination
sPath = bk.Positionals.Dest
if sPath == "" {
sPath, err = dis.NewSnapshotStagingDir("")
if err != nil {
return errSnapshotFailed.Wrap(err)
}
}
// create the directory (if it doesn't already exist)
logging.LogMessage(context.IOStream, "Creating staging directory")
err = os.Mkdir(sPath, fs.ModePerm)
if !os.IsExist(err) && err != nil {
return errSnapshotFailed.WithMessageF(err)
}
err = nil
}
context.Println(sPath)
logging.LogOperation(func() error {
// take a snapshot into the staging area!
backup := dis.Backup(context.IOStream, env.BackupDescription{
Dest: sPath,
})
// write out the report, ignoring any errors!
backup.WriteReport(context.IOStream)
return nil
}, context.IOStream, "Generating Backup")
// if we requested to only have a staging area, then we are done
if bk.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
// create the archive path
archivePath := bk.Positionals.Dest
if archivePath == "" {
archivePath = dis.NewSnapshotArchivePath("")
}
// and write everything into it!
// TODO: Should we move the open call to here?
var count int64
if err := logging.LogOperation(func() error {
context.IOStream.Println(archivePath)
count, err = targz.Package(archivePath, sPath, func(dst, src string) {
context.Println(dst)
})
return err
}, context.IOStream, "Writing backup archive"); err != nil {
return errSnapshotFailed.Wrap(err)
}
context.Printf("Wrote %d byte(s) to %s\n", count, archivePath)
return nil
}

108
cmd/snapshot.go
View file

@ -1,9 +1,8 @@
package cmd
import (
"fmt"
"io/fs"
"os"
"path/filepath"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/env"
@ -16,7 +15,8 @@ import (
var Snapshot wisski_distillery.Command = snapshot{}
type snapshot struct {
Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"`
Keepalive bool `short:"k" long:"keepalive" description:"Keep instance running while taking a backup. Might lead to inconsistent state"`
StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a snapshot archive, but only create a staging directory"`
Positionals struct {
Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"`
@ -46,65 +46,81 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
return err
}
// TODO: Allow skipping backups of individual parts and make them concurrent!
// start the snapshot and shutdown the instance (if requested)
logging.LogMessage(context.IOStream, "Creating snapshot of instance %s", bi.Positionals.Slug)
// create a new temporary directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
sPath, err := dis.NewSnapshotStagingDir(instance.Slug)
if err != nil {
return errSnapshotFailed.Wrap(err)
// determine the target path for the archive
var sPath string
if !bi.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
sPath, err = dis.NewSnapshotStagingDir(instance.Slug)
if err != nil {
return errSnapshotFailed.Wrap(err)
}
defer func() {
logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
os.RemoveAll(sPath)
}()
} else {
// staging mode: use dest as a destination
sPath = bi.Positionals.Dest
if sPath == "" {
sPath, err = dis.NewSnapshotStagingDir(instance.Slug)
if err != nil {
return errSnapshotFailed.Wrap(err)
}
}
// create the directory (if it doesn't already exist)
logging.LogMessage(context.IOStream, "Creating staging directory")
err = os.Mkdir(sPath, fs.ModePerm)
if !os.IsExist(err) && err != nil {
return errSnapshotFailed.WithMessageF(err)
}
err = nil
}
defer func() {
logging.LogMessage(context.IOStream, "Removing snapshot staging directory")
os.RemoveAll(sPath)
}()
context.Println(sPath)
// TODO: Allow skipping backups of individual parts and make them concurrent!
// take a snapshot into the staging area!
sreport := instance.Snapshot(context.IOStream, bi.Keepalive, sPath)
// write out the report!
logging.LogOperation(func() error {
sreport := instance.Snapshot(context.IOStream, env.SnapshotDescription{
Dest: sPath,
Keepalive: bi.Keepalive,
})
logging.LogOperation(func() error {
reportPath := filepath.Join(sPath, "report.txt")
context.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = fmt.Fprintf(report, "%#v\n", sreport)
return err
}, context.IOStream, "Writing snapshot report")
// write out the report, ignoring any errors!
sreport.WriteReport(context.IOStream)
return nil
}, context.IOStream, "Creating snapshot")
}, context.IOStream, "Generating Snapshot")
// copy everything into the final archive
finalPath := bi.Positionals.Dest
if finalPath == "" {
finalPath = dis.NewSnapshotArchivePath(instance.Slug)
// if we requested to only have a staging area, then we are done
if bi.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
if err := logging.LogOperation(func() error {
context.IOStream.Println(finalPath)
// create the archive path
archivePath := bi.Positionals.Dest
if archivePath == "" {
archivePath = dis.NewSnapshotArchivePath(instance.Slug)
}
targz.Package(finalPath, sPath, func(src string) {
context.Println(src)
// and write everything into it!
// TODO: Should we move the open call to here?
var count int64
if err := logging.LogOperation(func() error {
context.IOStream.Println(archivePath)
count, err = targz.Package(archivePath, sPath, func(dst, src string) {
context.Println(dst)
})
return err
}, context.IOStream, "Writing final backup"); err != nil {
}, context.IOStream, "Writing snapshot archive"); err != nil {
return errSnapshotFailed.Wrap(err)
}
context.Printf("Wrote %s\n", finalPath)
context.Printf("Wrote %d byte(s) to %s\n", count, archivePath)
return nil
}

0
cmd/prefix.go → cmd/update_prefix_config.go
View file

2
cmd/wdcli/main.go
View file

@ -45,7 +45,7 @@ func init() {
// backup & cron
wdcli.Register(cmd.Snapshot)
// wdcli.Register(cmd.BackupAll)
wdcli.Register(cmd.Backup)
wdcli.Register(cmd.Cron)
}

71
distillery/backup_all.sh
View file

@ -1,71 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
log_info " => Starting backup process. This might take a while. "
wait_for_sql
BACKUP_SLUG="$(date +%Y%m%dT%H%M%S)-$(randompw)"
BACKUP_INSTANCE_DIR="$DEPLOY_BACKUP_INPROGRESS_DIR/$BACKUP_SLUG"
BACKUP_FINAL_FILE="$DEPLOY_BACKUP_FINAL_DIR/$BACKUP_SLUG.tar.gz"
BACKUP_SQL_FILE="$BACKUP_INSTANCE_DIR/backup.sql"
BACKUP_TRIPLESTORE_DIR="$BACKUP_INSTANCE_DIR/triplestore"
BACKUP_TRIPLESTORE_SYSTEM="$BACKUP_TRIPLESTORE_DIR/system.nq"
BACKUP_FILESYSTEM_DIR="$BACKUP_INSTANCE_DIR/instances"
# create the backup directories
log_info " => Making '$BACKUP_INSTANCE_DIR'"
mkdir -p "$BACKUP_INSTANCE_DIR"
mkdir -p "$DEPLOY_BACKUP_FINAL_DIR"
function backup_everything() {
# backup the configuration
log_info " => Backing up configuration"
cp "$CONFIG_FILE" "$BACKUP_INSTANCE_DIR/.env" || true
# Backup sql (complete)
log_info " => Backing up the SQL database"
dockerized_mysqldump --all-databases > "$BACKUP_SQL_FILE" || true
# Backup triplestore (complete)
log_info " => Backing up Triplestore System"
mkdir -p "$BACKUP_TRIPLESTORE_DIR"
curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/SYSTEM/statements?infer=false" > "$BACKUP_TRIPLESTORE_SYSTEM" || true
# backup triplestore (individual)
for REPO in `grep -oP '(?<=#repositoryID> ")[^"]+' $BACKUP_TRIPLESTORE_SYSTEM`; do
log_info " => Backing up Triplestore Repository '$REPO'"
curl -X GET -H "Accept:application/n-quads" $GRAPHDB_AUTH_FLAGS "http://127.0.0.1:7200/repositories/$REPO/statements?infer=false" > "$BACKUP_TRIPLESTORE_DIR/repo_$REPO.nq" || true
done
# backup all the instances
log_info "=> Backing up instances"
for slug in $(sql_bookkeep_list); do
log_info "=> /bin/bash '$DIR/backup_instance.sh' '$slug' '$BACKUP_INSTANCE_DIR/${slug}.tar.gz'"
/bin/bash "$DIR/backup_instance.sh" "$slug" "$BACKUP_INSTANCE_DIR/${slug}.tar.gz" 2>&1 | tee "$BACKUP_INSTANCE_DIR/${slug}.log" || true
done
}
# do the entire backup
backup_everything 2>&1 | tee "$BACKUP_LOG_FILE.log"
# Package the backup into a .tar.gz
log_info " => Packaging '$BACKUP_FINAL_FILE'"
pushd "$BACKUP_INSTANCE_DIR" > /dev/null
tar --totals --checkpoint=10000 -zcf "$BACKUP_FINAL_FILE" .
popd > /dev/null
# Clean up the unpacked backup
log_info " => Cleaning up '$BACKUP_INSTANCE_DIR'"
rm -rf "$BACKUP_INSTANCE_DIR"
log_info " => Removing backups older than $MAX_BACKUP_AGE days"
find "$DEPLOY_BACKUP_FINAL_DIR" -type f -mtime "+$MAX_BACKUP_AGE" -print -exec rm -f {} \;

189
env/backup.go vendored Normal file
View file

@ -0,0 +1,189 @@
package env
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
// backupDescription is a description for a backup
type BackupDescription struct {
Dest string // destination path
}
// Snapshot represents the result of generating a snapshot
type Backup struct {
Description BackupDescription
// various error states, which are ignored when creating the snapshot
ErrPanic interface{}
SQLErr error
TSErr error
ConfigFileErr error
ConfigFilesManifest map[string]error
InstanceListErr error
InstancesManifest []Snapshot
}
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
backup.Description = description
// catch anything critical that happened during the snapshot
defer func() {
backup.ErrPanic = recover()
}()
backup.run(io, dis)
return
}
var errBackupSkipFile = errors.New("<file not found>")
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
// create a wait group, and message channel
wg := &sync.WaitGroup{}
messages := make(chan string, 4)
// backup the sql
wg.Add(1)
go func() {
defer wg.Done()
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
messages <- sqlPath
sql, err := os.Create(sqlPath)
if err != nil {
backup.SQLErr = err
return
}
defer sql.Close()
// directly store the result
backup.SQLErr = dis.SQL().BackupAll(io, sql)
}()
// backup the triplestore
wg.Add(1)
go func() {
defer wg.Done()
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
messages <- tsPath
// directly store the result
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
}()
// backup configuration files
wg.Add(1)
go func() {
defer wg.Done()
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
backup.ConfigFileErr = err
return
}
files := []string{
filepath.Join(dis.Config.DeployRoot, ".env"), // TODO: put the name of the configuration file somewhere
filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
dis.Config.SelfOverridesFile,
dis.Config.GlobalAuthorizedKeysFile,
}
backup.ConfigFilesManifest = make(map[string]error, len(files))
for _, src := range files {
if !fsx.IsFile(src) {
backup.ConfigFilesManifest[src] = errBackupSkipFile
continue
}
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
// copy the config file and store the error message
messages <- src
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
}
}()
// backup instances
wg.Add(1)
go func() {
defer wg.Done()
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
backup.InstanceListErr = err
return
}
// list all instances
instances, err := dis.AllInstances()
if err != nil {
backup.InstanceListErr = err
return
}
iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
backup.InstancesManifest = make([]Snapshot, len(instances))
for i, instance := range instances {
backup.InstancesManifest[i] = func() Snapshot {
dir := filepath.Join(instancesBackupDir, instance.Slug)
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
return Snapshot{
ErrPanic: err,
}
}
messages <- dir
return instance.Snapshot(iochild, SnapshotDescription{
Dest: dir,
})
}()
}
}()
// wait for the group, then close the message channel.
go func() {
wg.Wait()
close(messages)
}()
// print out all the messages
for message := range messages {
io.Println(message)
}
}
// WriteReport writes out the report belonging to this backup.
// It is a separate function, to allow writing it indepenently of the rest.
func (backup Backup) WriteReport(io stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
io.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = fmt.Fprintf(report, "%#v\n", backup)
return err
}, io, "Writing backup report")
}

16
env/component_sql.go vendored
View file

@ -103,7 +103,21 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code"
func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--database", database)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}
if code != 0 {
return errSQLBackup
}
return nil
}
// BackupAll makes a backup of all sql databases
func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}

57
env/component_triplestore.go vendored
View file

@ -8,6 +8,7 @@ import (
"io/fs"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"time"
@ -237,6 +238,62 @@ func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error)
return io.Copy(dst, res.Body)
}
type Repository struct {
ID string `json:"id"`
Title string `json:"title"`
URI string `json:"uri"`
Type string `json:"type"`
SesameType string `json:"sesameType"`
Location string `json:"location"`
Readable bool `json:"readable"`
Writable bool `json:"writable"`
Local bool `json:"local"`
}
func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error) {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
if err != nil {
return nil, err
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(&repos)
return
}
// TriplestoreBackup backs up every graphdb instance into dst
func (ts TriplestoreComponent) BackupAll(dst string) error {
// list all the repositories
repos, err := ts.listRepositories()
if err != nil {
return err
}
// create the base directory
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
return err
}
// iterate over all the repositories
for _, repo := range repos {
if rErr := (func(repo Repository) error {
name := filepath.Join(dst, repo.ID+".nq")
dest, err := os.Create(name)
if err != nil {
return err
}
defer dest.Close()
_, err = ts.Backup(dest, repo.ID)
return err
}(repo)); err == nil && rErr != nil {
err = rErr
}
}
return err
}
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {

104
env/snapshot.go vendored
View file

@ -8,6 +8,7 @@ import (
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/FAU-CDI/wisski-distillery/internal/password"
@ -46,6 +47,11 @@ func (dis Distillery) NewSnapshotArchivePath(prefix string) (path string) {
// The name is guaranteed to be unique within this process.
func (Distillery) newSnapshotName(prefix string) string {
suffix, _ := password.Password(64) // silently ignore any errors!
if prefix == "" {
prefix = "backup"
} else {
prefix = "snapshot-" + prefix
}
return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
}
@ -62,36 +68,56 @@ func (dis Distillery) NewSnapshotStagingDir(prefix string) (path string, err err
return
}
type SnapshotReport struct {
Keepalive bool // was the instance alive while running the snapshot?
Panic interface{} // was there a panic?
// SnapshotDescription is a description for a snapshot
type SnapshotDescription struct {
Dest string // destination path
Keepalive bool // should we keep the instance alive while making the snapshot?
}
// errors for the various components of the Snapshot
StopErr error
StartErr error
BookkeepingErr error
FilesystemErr error
TriplestoreErr error
SQLErr error
// Snapshot represents the result of generating a snapshot
type Snapshot struct {
Description SnapshotDescription
Instance bookkeeping.Instance
// various error states, which are ignored when creating the snapshot
ErrPanic interface{} // panic, if any
ErrStart error
ErrStop error
ErrBookkeep error
ErrFilesystem error
ErrTriplestore error
ErrSSQL error
}
// Snapshot creates a new snapshot of this instance into dest
func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest string) (report SnapshotReport) {
func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
// setup the snapshot
snapshot.Description = desc
snapshot.Instance = instance.Instance
// catch anything critical that happened during the snapshot
defer func() {
report.Panic = recover()
snapshot.ErrPanic = recover()
}()
// and do the create!
snapshot.create(io, instance)
return
}
func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
stack := instance.Stack()
// stop the instance (unless it was explicitly asked to not do so!)
report.Keepalive = keepalive
if !keepalive {
if !snapshot.Description.Keepalive {
logging.LogMessage(io, "Stopping instance")
report.StopErr = stack.Down(io)
snapshot.ErrStop = stack.Down(io)
defer func() {
logging.LogMessage(io, "Starting instance")
report.StartErr = stack.Up(io)
snapshot.ErrStart = stack.Up(io)
}()
}
@ -104,19 +130,19 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
go func() {
defer wg.Done()
bkPath := filepath.Join(dest, "bookkeeping.txt")
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
messages <- bkPath
info, err := os.Create(bkPath)
if err != nil {
report.BookkeepingErr = err
snapshot.ErrBookkeep = err
return
}
defer info.Close()
// print whatever is in the database
// TODO: This should be sql code, maybe gorm can do that?
_, report.BookkeepingErr = fmt.Fprintf(info, "%#v\n", instance.Instance)
_, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance)
}()
// backup the filesystem
@ -124,14 +150,14 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
go func() {
defer wg.Done()
fsPath := filepath.Join(dest, filepath.Base(instance.FilesystemBase))
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
report.FilesystemErr = err
snapshot.ErrFilesystem = err
return
}
// copy over whatever is in the base directory
report.FilesystemErr = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
messages <- dst
})
}()
@ -141,17 +167,17 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
go func() {
defer wg.Done()
tsPath := filepath.Join(dest, instance.GraphDBRepository+".nq")
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
messages <- tsPath
nquads, err := os.Create(tsPath)
if err != nil {
report.TriplestoreErr = err
snapshot.ErrTriplestore = err
}
defer nquads.Close()
// directly store the result
_, report.TriplestoreErr = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
_, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
}()
// backup the sql database
@ -159,18 +185,18 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
go func() {
defer wg.Done()
sqlPath := filepath.Join(dest, instance.SqlDatabase+".sql")
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
messages <- sqlPath
sql, err := os.Create(sqlPath)
if err != nil {
report.SQLErr = err
snapshot.ErrSSQL = err
return
}
defer sql.Close()
// directly store the result
report.SQLErr = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
}()
// TODO: Backup the docker image
@ -185,6 +211,24 @@ func (instance Instance) Snapshot(io stream.IOStream, keepalive bool, dest strin
for message := range messages {
io.Println(message)
}
return
}
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
io.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = fmt.Fprintf(report, "%#v\n", snapshot)
return err
}, io, "Writing snapshot report")
}

30
internal/targz/targz.go
View file

@ -11,11 +11,11 @@ import (
// Package packages the directory src into dst.
// onCopy, when not nil, is called for each file being copied into the archive.
func Package(dst, src string, onCopy func(src string)) error {
func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
// create the target archive
archive, err := os.Create(dst)
if err != nil {
return err
return 0, err
}
defer archive.Close()
@ -28,21 +28,28 @@ func Package(dst, src string, onCopy func(src string)) error {
defer tarHandle.Close()
// and walk through it!
return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if onCopy != nil {
onCopy(path)
}
err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// determine the relative path
var relpath string
relpath, err = filepath.Rel(src, path)
if err != nil {
return err
}
if onCopy != nil {
onCopy(relpath, path)
}
// create a file info header!
tInfo, err := tar.FileInfoHeader(info, path)
tInfo, err := tar.FileInfoHeader(info, relpath)
if err != nil {
return err
}
tInfo.Name = filepath.ToSlash(path)
tInfo.Name = filepath.ToSlash(relpath)
// write it!
if err := tarHandle.WriteHeader(tInfo); err != nil {
@ -62,8 +69,9 @@ func Package(dst, src string, onCopy func(src string)) error {
defer handle.Close()
// and copy it into the archive
_, err = io.Copy(tarHandle, handle)
ccount, err := io.Copy(tarHandle, handle)
count += ccount
return err
})
return
}