'wdcli backup': Move to separate package
This commit is contained in:
Tom Wiesing
parent
5cd5ae9be2
commit
822c70cd69
11 changed files with 493 additions and 380 deletions
|
|
@ -5,17 +5,17 @@ import (
|
|||
"os"
|
||||
|
||||
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/backup"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
|
||||
"github.com/tkw1536/goprogram/exit"
|
||||
)
|
||||
|
||||
// Backup is the 'backup' command
|
||||
var Backup wisski_distillery.Command = backup{}
|
||||
var Backup wisski_distillery.Command = backupC{}
|
||||
|
||||
type backup struct {
|
||||
type backupC struct {
|
||||
NoPrune bool `short:"n" long:"no-prune" description:"Do not prune older backup archives"`
|
||||
StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"`
|
||||
Positionals struct {
|
||||
|
|
@ -23,7 +23,7 @@ type backup struct {
|
|||
} `positional-args:"true"`
|
||||
}
|
||||
|
||||
func (backup) Description() wisski_distillery.Description {
|
||||
func (backupC) Description() wisski_distillery.Description {
|
||||
return wisski_distillery.Description{
|
||||
Requirements: core.Requirements{
|
||||
NeedsDistillery: true,
|
||||
|
|
@ -38,7 +38,7 @@ var errBackupFailed = exit.Error{
|
|||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
func (bk backup) Run(context wisski_distillery.Context) error {
|
||||
func (bk backupC) Run(context wisski_distillery.Context) error {
|
||||
dis := context.Environment
|
||||
var err error
|
||||
|
||||
|
|
@ -82,14 +82,11 @@ func (bk backup) Run(context wisski_distillery.Context) error {
|
|||
context.Println(sPath)
|
||||
|
||||
logging.LogOperation(func() error {
|
||||
// take a snapshot into the staging area!
|
||||
backup := dis.Backup(context.IOStream, wisski.BackupDescription{
|
||||
backup := backup.New(context.IOStream, dis, backup.Description{
|
||||
Dest: sPath,
|
||||
Auto: bk.Positionals.Dest == "",
|
||||
})
|
||||
|
||||
// write out the report, ignoring any errors!
|
||||
backup.WriteReport(context.IOStream)
|
||||
|
||||
return nil
|
||||
}, context.IOStream, "Generating Backup")
|
||||
|
||||
|
|
|
|||
141
internal/backup/backup.go
Normal file
141
internal/backup/backup.go
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
// Package backup implements Distillery backups.
|
||||
package backup
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// New create a new Backup
|
||||
func New(io stream.IOStream, dis *wisski.Distillery, description Description) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
backup.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
backup.StartTime = time.Now()
|
||||
backup.run(io, dis)
|
||||
backup.EndTime = time.Now()
|
||||
|
||||
return nil
|
||||
}, io, "Writing backup files")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type backupResult struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
func (backup *Backup) run(io stream.IOStream, dis *wisski.Distillery) {
|
||||
|
||||
backups := dis.Backupable()
|
||||
|
||||
files := make(chan string, len(backups)) // channel for files being added into the backups
|
||||
results := make(chan backupResult, len(backups)) // channel for results to be stored into
|
||||
backup.ComponentErrors = make(map[string]error, len(backups))
|
||||
|
||||
wg := &sync.WaitGroup{} // to wait for the results
|
||||
wg.Add(len(backups)) // tell the group about all the operations
|
||||
for _, bc := range backups {
|
||||
go func(bc component.Backupable, context component.BackupContext) {
|
||||
defer wg.Done()
|
||||
|
||||
// make the backup and store the result
|
||||
results <- backupResult{
|
||||
name: bc.Name(),
|
||||
err: bc.Backup(context),
|
||||
}
|
||||
}(bc, &context{
|
||||
io: io,
|
||||
dst: filepath.Join(backup.Description.Dest, bc.BackupName()),
|
||||
files: files,
|
||||
})
|
||||
}
|
||||
|
||||
// backup instances
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
|
||||
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
// list all instances
|
||||
instances, err := dis.Instances().All()
|
||||
if err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
backup.InstanceSnapshots = make([]wisski.Snapshot, len(instances))
|
||||
for i, instance := range instances {
|
||||
backup.InstanceSnapshots[i] = func() wisski.Snapshot {
|
||||
dir := filepath.Join(instancesBackupDir, instance.Slug)
|
||||
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
|
||||
return wisski.Snapshot{
|
||||
ErrPanic: err,
|
||||
}
|
||||
}
|
||||
|
||||
files <- dir
|
||||
return dis.Snapshot(instance, io.NonInteractive(), wisski.SnapshotDescription{
|
||||
Dest: dir,
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
// finish processing all the results as soon as the group is done.
|
||||
go func() {
|
||||
defer close(results)
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
// finish the message processing once results are finished.
|
||||
go func() {
|
||||
defer close(files) // no more file processing!
|
||||
for result := range results {
|
||||
backup.ComponentErrors[result.name] = result.err
|
||||
}
|
||||
}()
|
||||
|
||||
for file := range files {
|
||||
// get the relative path to the root of the manifest.
|
||||
// nothing *should* go wrong, but in case it does, use the original path.
|
||||
path, err := filepath.Rel(backup.Description.Dest, file)
|
||||
if err != nil {
|
||||
path = file
|
||||
}
|
||||
|
||||
// write it to the command line
|
||||
// and also add it to the manifest
|
||||
io.Printf("\033[2K\r%s", path)
|
||||
backup.Manifest = append(backup.Manifest, path)
|
||||
}
|
||||
slices.Sort(backup.Manifest) // backup the manifest
|
||||
io.Println("")
|
||||
|
||||
// sort the instances manifest
|
||||
slices.SortFunc(backup.InstanceSnapshots, func(a, b wisski.Snapshot) bool {
|
||||
return a.Instance.Slug < b.Instance.Slug
|
||||
})
|
||||
}
|
||||
98
internal/backup/context.go
Normal file
98
internal/backup/context.go
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// context implements [components.BackupContext]
|
||||
type context struct {
|
||||
io stream.IOStream
|
||||
dst string // destination directory
|
||||
files chan string // files channel
|
||||
}
|
||||
|
||||
func (bc *context) sendPath(path string) {
|
||||
|
||||
// resolve the path, or bail out!
|
||||
// TODO: Use the relative path here!
|
||||
dst, err := bc.resolve(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
bc.files <- dst
|
||||
}
|
||||
|
||||
func (bc *context) IO() stream.IOStream {
|
||||
return bc.io
|
||||
}
|
||||
|
||||
var errResolveAbsolute = errors.New("resolve: path must be relative")
|
||||
|
||||
func (bc *context) resolve(path string) (dest string, err error) {
|
||||
if path == "" {
|
||||
return bc.dst, nil
|
||||
}
|
||||
if filepath.IsAbs(path) {
|
||||
return "", errResolveAbsolute
|
||||
}
|
||||
return filepath.Join(bc.dst, path), nil
|
||||
}
|
||||
|
||||
func (bc *context) AddDirectory(path string, op func() error) error {
|
||||
// resolve the path!
|
||||
dst, err := bc.resolve(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// run the make directory
|
||||
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// tell the files that we are creating it!
|
||||
bc.sendPath(path)
|
||||
|
||||
// and run the files!
|
||||
// TODO: Add to manifest of some sort
|
||||
return op()
|
||||
}
|
||||
|
||||
func (bc *context) CopyFile(dst, src string) error {
|
||||
dstPath, err := bc.resolve(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bc.sendPath(dst)
|
||||
return fsx.CopyFile(dstPath, src)
|
||||
}
|
||||
|
||||
func (bc *context) AddFile(path string, op func(file io.Writer) error) error {
|
||||
// resolve the path!
|
||||
dst, err := bc.resolve(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the file
|
||||
file, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// tell them that we are creating it!
|
||||
bc.sendPath(path)
|
||||
|
||||
// and do whatever they wanted to do
|
||||
// TODO: Add to the manifest of some sort
|
||||
return op(file)
|
||||
}
|
||||
115
internal/backup/report.go
Normal file
115
internal/backup/report.go
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
package backup
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// Description provides a description for a backup
|
||||
type Description struct {
|
||||
Dest string // Destination path
|
||||
Auto bool // Was the path created automatically?
|
||||
}
|
||||
|
||||
// Backup describes a backup
|
||||
type Backup struct {
|
||||
Description Description
|
||||
|
||||
// Start and End Time of the backup
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
// errors for the various components
|
||||
ComponentErrors map[string]error
|
||||
|
||||
// TODO: Make this proper
|
||||
ConfigFileErr error
|
||||
|
||||
// Snapshots containing instances
|
||||
InstanceListErr error
|
||||
InstanceSnapshots []wisski.Snapshot
|
||||
|
||||
// List of files included
|
||||
Manifest []string
|
||||
}
|
||||
|
||||
// Strings turns this backup into a string for the BackupReport.
|
||||
func (backup Backup) String() string {
|
||||
var builder strings.Builder
|
||||
backup.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report formats a report for this backup, and writes it into Writer.
|
||||
func (backup Backup) Report(w io.Writer) (int, error) {
|
||||
cw := countwriter.NewCountWriter(w)
|
||||
|
||||
encoder := json.NewEncoder(cw)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(cw, "======= Backup =======\n")
|
||||
|
||||
fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
|
||||
fmt.Fprintf(cw, "End: %s\n", backup.EndTime)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Description =======\n")
|
||||
encoder.Encode(backup.Description)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Errors =======\n")
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Snapshots =======\n")
|
||||
for _, s := range backup.InstanceSnapshots {
|
||||
io.WriteString(cw, s.String())
|
||||
io.WriteString(cw, "\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "======= Manifest =======\n")
|
||||
for _, file := range backup.Manifest {
|
||||
io.WriteString(cw, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
return cw.Sum()
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this backup.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (backup Backup) WriteReport(io stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
|
||||
io.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := os.Create(reportPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = report.WriteString(backup.String())
|
||||
return err
|
||||
}, io, "Writing backup report")
|
||||
}
|
||||
43
internal/component/backup.go
Normal file
43
internal/component/backup.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package component
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// Backupable represents a component with a Backup method
|
||||
type Backupable interface {
|
||||
Component
|
||||
|
||||
// BackupName returns a new name to be used as an argument for path.
|
||||
BackupName() string
|
||||
|
||||
// Backup backs up this component into the destination path path
|
||||
Backup(context BackupContext) error
|
||||
}
|
||||
|
||||
// BackupContext is the context for backups
|
||||
type BackupContext interface {
|
||||
// IO returns the input output stream belonging to this backup file
|
||||
IO() stream.IOStream
|
||||
|
||||
// Name creates a new directory inside the destination.
|
||||
// Passing the empty path creates the destination as a directory.
|
||||
//
|
||||
// It then allows op to fill the file.
|
||||
AddDirectory(path string, op func() error) error
|
||||
|
||||
// CopyFile copies a file from source to dst.
|
||||
CopyFile(dest, src string) error
|
||||
|
||||
// AddFile creates a new file at the provided path inside the destination.
|
||||
// Passing the empty path creates the destination as a file.
|
||||
//
|
||||
// It then allows op to write to the file.
|
||||
//
|
||||
// The op function must not retain file.
|
||||
// The underlying file does not need to be closed.
|
||||
// AddFile will not return before op has returned.
|
||||
AddFile(path string, op func(file io.Writer) error) error
|
||||
}
|
||||
|
|
@ -3,7 +3,6 @@ package component
|
|||
|
||||
import (
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// Component represents a logical subsystem of the distillery.
|
||||
|
|
@ -46,20 +45,6 @@ type Installable interface {
|
|||
Context(parent InstallationContext) InstallationContext
|
||||
}
|
||||
|
||||
// Backupable represents a component with a Backup method
|
||||
type Backupable interface {
|
||||
Component
|
||||
|
||||
// BackupName returns a new name to be used as an argument for path.
|
||||
BackupName() string
|
||||
|
||||
// Backup backs up this component into the destination path path.
|
||||
//
|
||||
// The destination path may be a folder or directory, depending on the component.
|
||||
// The destination path does not need to exist.
|
||||
Backup(io stream.IOStream, path string) error
|
||||
}
|
||||
|
||||
// ComponentBase implements base functionality for a component
|
||||
type ComponentBase struct {
|
||||
Dir string // Dir is the directory this component lives in
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
package control
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
func (*Control) BackupName() string {
|
||||
|
|
@ -14,27 +11,18 @@ func (*Control) BackupName() string {
|
|||
}
|
||||
|
||||
// Backup backups all control plane configuration files into dest
|
||||
func (control *Control) Backup(io stream.IOStream, dest string) error {
|
||||
// create the destination directory, TODO: outsource this
|
||||
if err := os.Mkdir(dest, fs.ModeDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func (control *Control) Backup(context component.BackupContext) error {
|
||||
files := control.backupFiles()
|
||||
for _, src := range files {
|
||||
dst := filepath.Join(dest, filepath.Base(src)) // destination path
|
||||
|
||||
// if the src file does not exist, don't copy it!
|
||||
if !fsx.IsFile(src) { // TODO: log this somewhere
|
||||
continue
|
||||
return context.AddDirectory("", func() error {
|
||||
for _, src := range files {
|
||||
name := filepath.Base(src)
|
||||
if err := context.CopyFile(name, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := fsx.CopyFile(dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// backupfiles lists the files to be backed up.
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ package sql
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"io"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
|
||||
|
|
@ -14,22 +14,17 @@ func (*SQL) BackupName() string {
|
|||
}
|
||||
|
||||
// Backup makes a backup of all SQL databases into the path dest.
|
||||
func (sql *SQL) Backup(io stream.IOStream, dest string) error {
|
||||
// open the file, TODO: Outsource this to context
|
||||
writer, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer writer.Close()
|
||||
func (sql *SQL) Backup(context component.BackupContext) error {
|
||||
return context.AddFile("", func(file io.Writer) error {
|
||||
io := context.IO().Streams(file, nil, nil, 0).NonInteractive()
|
||||
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errSQLBackup
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// run sqldump
|
||||
io = io.Streams(writer, nil, nil, 0).NonInteractive()
|
||||
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errSQLBackup
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,48 +2,34 @@ package triplestore
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"io"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
func (ts *Triplestore) BackupName() string { return "triplestore" }
|
||||
|
||||
// Backup makes a backup of all Triplestore repositories databases into the path dest.
|
||||
func (ts *Triplestore) Backup(io stream.IOStream, dest string) error {
|
||||
func (ts *Triplestore) Backup(context component.BackupContext) error {
|
||||
|
||||
// list all the repositories
|
||||
// list all the directories
|
||||
repos, err := ts.listRepositories()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the base directory, todo: outsource this
|
||||
if err := os.Mkdir(dest, fs.ModeDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// iterate over all the repositories
|
||||
for _, repo := range repos {
|
||||
if rErr := (func(repo Repository) error {
|
||||
name := filepath.Join(dest, repo.ID+".nq")
|
||||
|
||||
// todo: outsource this
|
||||
dest, err := os.Create(name)
|
||||
if err != nil {
|
||||
// then backup each file separatly
|
||||
return context.AddDirectory("", func() error {
|
||||
for _, repo := range repos {
|
||||
if err := context.AddFile(repo.ID+".nq", func(file io.Writer) error {
|
||||
_, err := ts.Snapshot(file, repo.ID)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
defer dest.Close()
|
||||
|
||||
_, err = ts.Snapshot(dest, repo.ID)
|
||||
return err
|
||||
}(repo)); err == nil && rErr != nil {
|
||||
err = rErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
|
||||
|
|
|
|||
|
|
@ -1,288 +1 @@
|
|||
package wisski
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// backupDescription is a description for a backup
|
||||
type BackupDescription struct {
|
||||
Dest string // destination path
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Backup struct {
|
||||
Description BackupDescription
|
||||
|
||||
// Start and End Time of the backup
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
// errors for the various components
|
||||
ComponentErrors map[string]error
|
||||
|
||||
// SQL and triplestore errors
|
||||
TSErr error
|
||||
|
||||
// TODO: Make this proper
|
||||
ConfigFileErr error
|
||||
|
||||
// Snapshots containing instances
|
||||
InstanceListErr error
|
||||
InstanceSnapshots []Snapshot
|
||||
|
||||
// List of files included
|
||||
Manifest []string
|
||||
}
|
||||
|
||||
func (backup Backup) String() string {
|
||||
var builder strings.Builder
|
||||
backup.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report writes a report from backup into w
|
||||
func (backup Backup) Report(w io.Writer) (int, error) {
|
||||
cw := countwriter.NewCountWriter(w)
|
||||
|
||||
encoder := json.NewEncoder(cw)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(cw, "======= Backup =======\n")
|
||||
|
||||
fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
|
||||
fmt.Fprintf(cw, "End: %s\n", backup.EndTime)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Description =======\n")
|
||||
encoder.Encode(backup.Description)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Errors =======\n")
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Snapshots =======\n")
|
||||
for _, s := range backup.InstanceSnapshots {
|
||||
io.WriteString(cw, s.String())
|
||||
io.WriteString(cw, "\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "======= Manifest =======\n")
|
||||
for _, file := range backup.Manifest {
|
||||
io.WriteString(cw, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
return cw.Sum()
|
||||
}
|
||||
|
||||
// Backup makes a makes of the entire distillery.
|
||||
// To make a backup, all [BackupComponents] will be invoked.
|
||||
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
backup.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
backup.StartTime = time.Now()
|
||||
backup.run(io, dis)
|
||||
backup.EndTime = time.Now()
|
||||
|
||||
return nil
|
||||
}, io, "Writing backup files")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type backupResult struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
|
||||
|
||||
backups := dis.Backupable()
|
||||
|
||||
files := make(chan string, len(backups)) // channel for files being added into the backups
|
||||
results := make(chan backupResult, len(backups)) // channel for results to be stored into
|
||||
backup.ComponentErrors = make(map[string]error, len(backups))
|
||||
|
||||
wg := &sync.WaitGroup{} // to wait for the results
|
||||
wg.Add(len(backups))
|
||||
for _, bc := range backups {
|
||||
go func(bc component.Backupable) {
|
||||
defer wg.Done()
|
||||
|
||||
// find the backup destination
|
||||
dest := filepath.Join(backup.Description.Dest, bc.BackupName())
|
||||
files <- dest
|
||||
|
||||
// make the backup and send the result!
|
||||
results <- backupResult{
|
||||
name: bc.Name(),
|
||||
err: bc.Backup(io, dest),
|
||||
}
|
||||
}(bc)
|
||||
}
|
||||
|
||||
// backup instances
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
|
||||
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
// list all instances
|
||||
instances, err := dis.Instances().All()
|
||||
if err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
backup.InstanceSnapshots = make([]Snapshot, len(instances))
|
||||
for i, instance := range instances {
|
||||
backup.InstanceSnapshots[i] = func() Snapshot {
|
||||
dir := filepath.Join(instancesBackupDir, instance.Slug)
|
||||
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
|
||||
return Snapshot{
|
||||
ErrPanic: err,
|
||||
}
|
||||
}
|
||||
|
||||
files <- dir
|
||||
return dis.Snapshot(instance, io.NonInteractive(), SnapshotDescription{
|
||||
Dest: dir,
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
// finish processing all the results as soon as the group is done.
|
||||
go func() {
|
||||
defer close(results)
|
||||
wg.Wait()
|
||||
}()
|
||||
|
||||
// finish the message processing once results are finished.
|
||||
go func() {
|
||||
defer close(files) // no more file processing!
|
||||
for result := range results {
|
||||
backup.ComponentErrors[result.name] = result.err
|
||||
}
|
||||
}()
|
||||
|
||||
for file := range files {
|
||||
// get the relative path to the root of the manifest.
|
||||
// nothing *should* go wrong, but in case it does, use the original path.
|
||||
path, err := filepath.Rel(backup.Description.Dest, file)
|
||||
if err != nil {
|
||||
path = file
|
||||
}
|
||||
|
||||
// write it to the command line
|
||||
// and also add it to the manifest
|
||||
io.Printf("\033[2K\r%s", path)
|
||||
backup.Manifest = append(backup.Manifest, path)
|
||||
}
|
||||
slices.Sort(backup.Manifest) // backup the manifest
|
||||
io.Println("")
|
||||
|
||||
// sort the instances manifest
|
||||
slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
|
||||
return a.Instance.Slug < b.Instance.Slug
|
||||
})
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this backup.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (backup Backup) WriteReport(io stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
|
||||
io.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := os.Create(reportPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = report.WriteString(backup.String())
|
||||
return err
|
||||
}, io, "Writing backup report")
|
||||
}
|
||||
|
||||
// ShouldPrune determines if a file with the provided modtime
|
||||
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
|
||||
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
|
||||
}
|
||||
|
||||
// PruneBackups prunes all backups older than the maximum backup age
|
||||
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
|
||||
sPath := dis.SnapshotsArchivePath()
|
||||
|
||||
// list all the files
|
||||
entries, err := os.ReadDir(sPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
// skip directories
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// grab info about the file
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if it should be pruned!
|
||||
if !dis.ShouldPrune(info.ModTime()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// assemble path, and then remove the file!
|
||||
path := filepath.Join(sPath, entry.Name())
|
||||
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
52
internal/wisski/backup_prune.go
Normal file
52
internal/wisski/backup_prune.go
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
package wisski
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// ShouldPrune determines if a file with the provided modtime
|
||||
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
|
||||
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
|
||||
}
|
||||
|
||||
// PruneBackups prunes all backups older than the maximum backup age
|
||||
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
|
||||
sPath := dis.SnapshotsArchivePath()
|
||||
|
||||
// list all the files
|
||||
entries, err := os.ReadDir(sPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
// skip directories
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// grab info about the file
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if it should be pruned!
|
||||
if !dis.ShouldPrune(info.ModTime()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// assemble path, and then remove the file!
|
||||
path := filepath.Join(sPath, entry.Name())
|
||||
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue