snapshots: Handle as separate components
This commit is contained in:
Tom Wiesing
parent
698f04e13e
commit
3b112f1b8e
27 changed files with 960 additions and 789 deletions
170
internal/component/snapshots/backup.go
Normal file
170
internal/component/snapshots/backup.go
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
package snapshots
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/tkw1536/goprogram/status"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// Backup describes a backup
|
||||
type Backup struct {
|
||||
Description BackupDescription
|
||||
|
||||
// Start and End Time of the backup
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
// errors for the various components
|
||||
ComponentErrors map[string]error
|
||||
|
||||
// TODO: Make this proper
|
||||
ConfigFileErr error
|
||||
|
||||
// Snapshots containing instances
|
||||
InstanceListErr error
|
||||
InstanceSnapshots []Snapshot
|
||||
|
||||
// List of files included
|
||||
WithManifest
|
||||
}
|
||||
|
||||
// BackupDescription provides a description for a backup
|
||||
type BackupDescription struct {
|
||||
Dest string // Destination path
|
||||
Auto bool // Was the path created automatically?
|
||||
|
||||
ConcurrentSnapshots int // maximum number of concurrent snapshots
|
||||
}
|
||||
|
||||
// New create a new Backup
|
||||
func (manager *Manager) NewBackup(io stream.IOStream, description BackupDescription) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
backup.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
backup.StartTime = time.Now().UTC()
|
||||
backup.run(io, manager)
|
||||
backup.EndTime = time.Now().UTC()
|
||||
|
||||
return nil
|
||||
}, io, "Writing backup files")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (backup *Backup) run(ios stream.IOStream, manager *Manager) {
|
||||
// create a manifest
|
||||
manifest, done := backup.handleManifest(backup.Description.Dest)
|
||||
defer done()
|
||||
|
||||
// create a new status display
|
||||
backups := manager.Backupable
|
||||
backup.ComponentErrors = make(map[string]error, len(backups))
|
||||
|
||||
// Component backup tasks
|
||||
logging.LogOperation(func() error {
|
||||
st := status.NewWithCompat(ios.Stdout, 0)
|
||||
st.Start()
|
||||
defer st.Stop()
|
||||
|
||||
errors := status.Group[component.Backupable, error]{
|
||||
PrefixString: func(item component.Backupable, index int) string {
|
||||
return fmt.Sprintf("[backup %q]: ", item.Name())
|
||||
},
|
||||
PrefixAlign: true,
|
||||
|
||||
Handler: func(bc component.Backupable, index int, writer io.Writer) error {
|
||||
return bc.Backup(
|
||||
component.NewStagingContext(
|
||||
manager.Environment,
|
||||
stream.NewIOStream(writer, writer, nil, 0),
|
||||
filepath.Join(backup.Description.Dest, bc.BackupName()),
|
||||
manifest,
|
||||
),
|
||||
)
|
||||
},
|
||||
|
||||
ResultString: status.DefaultErrorString[component.Backupable],
|
||||
}.Use(st, backups)
|
||||
|
||||
for i, bc := range backups {
|
||||
backup.ComponentErrors[bc.Name()] = errors[i]
|
||||
}
|
||||
|
||||
return nil
|
||||
}, ios, "Backing up core components")
|
||||
|
||||
// backup instances
|
||||
logging.LogOperation(func() error {
|
||||
st := status.NewWithCompat(ios.Stdout, 0)
|
||||
st.Start()
|
||||
defer st.Stop()
|
||||
|
||||
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
|
||||
if err := manager.Environment.Mkdir(instancesBackupDir, environment.DefaultDirPerm); err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
// list all instances
|
||||
wissKIs, err := manager.Instances.All()
|
||||
if err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
// make a backup of the snapshots
|
||||
backup.InstanceSnapshots = status.Group[instances.WissKI, Snapshot]{
|
||||
PrefixString: func(item instances.WissKI, index int) string {
|
||||
return fmt.Sprintf("[snapshot %q]: ", item.Slug)
|
||||
},
|
||||
PrefixAlign: true,
|
||||
|
||||
Handler: func(instance instances.WissKI, index int, writer io.Writer) Snapshot {
|
||||
dir := filepath.Join(instancesBackupDir, instance.Slug)
|
||||
if err := manager.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil {
|
||||
return Snapshot{
|
||||
ErrPanic: err,
|
||||
}
|
||||
}
|
||||
|
||||
manifest <- dir
|
||||
|
||||
return manager.NewSnapshot(instance, stream.NewIOStream(writer, writer, nil, 0), SnapshotDescription{
|
||||
Dest: dir,
|
||||
})
|
||||
},
|
||||
ResultString: func(res Snapshot, item instances.WissKI, index int) string {
|
||||
return "done"
|
||||
},
|
||||
WaitString: status.DefaultWaitString[instances.WissKI],
|
||||
HandlerLimit: backup.Description.ConcurrentSnapshots,
|
||||
}.Use(st, wissKIs)
|
||||
|
||||
// sort the instances
|
||||
slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
|
||||
return a.Instance.Slug < b.Instance.Slug
|
||||
})
|
||||
|
||||
return nil
|
||||
}, ios, "Creating instance snapshots")
|
||||
|
||||
}
|
||||
|
|
@ -6,8 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/password"
|
||||
|
|
@ -16,9 +15,10 @@ import (
|
|||
// Manager manages snapshots and backups
|
||||
type Manager struct {
|
||||
component.ComponentBase
|
||||
Instances *instances.Instances
|
||||
|
||||
TS *triplestore.Triplestore
|
||||
SQL *sql.SQL
|
||||
Snapshotable []component.Snapshotable
|
||||
Backupable []component.Backupable
|
||||
}
|
||||
|
||||
func (Manager) Name() string { return "snapshots" }
|
||||
|
|
@ -68,6 +68,7 @@ func (*Manager) newSnapshotName(prefix string) string {
|
|||
func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
|
||||
for path == "" || environment.IsExist(err) {
|
||||
path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
|
||||
fmt.Println("path =>", prefix, "err => ", err)
|
||||
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
|
||||
}
|
||||
if err != nil {
|
||||
33
internal/component/snapshots/manifest.go
Normal file
33
internal/component/snapshots/manifest.go
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
package snapshots
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type WithManifest struct {
|
||||
Manifest []string
|
||||
}
|
||||
|
||||
func (wm *WithManifest) handleManifest(dest string) (chan<- string, func()) {
|
||||
manifest := make(chan string)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
|
||||
for file := range manifest {
|
||||
// get the relative path to the root of the manifest.
|
||||
// nothing *should* go wrong, but in case it does, use the original path.
|
||||
path, err := filepath.Rel(dest, file)
|
||||
if err != nil {
|
||||
path = file
|
||||
}
|
||||
|
||||
// add the manifest
|
||||
wm.Manifest = append(wm.Manifest, path)
|
||||
}
|
||||
}()
|
||||
return manifest, func() {
|
||||
close(manifest)
|
||||
<-done
|
||||
}
|
||||
}
|
||||
154
internal/component/snapshots/report.go
Normal file
154
internal/component/snapshots/report.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package snapshots
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
func (snapshot Snapshot) String() string {
|
||||
var builder strings.Builder
|
||||
snapshot.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report writes a report from snapshot into w
|
||||
func (snapshot Snapshot) Report(w io.Writer) (int, error) {
|
||||
ww := countwriter.NewCountWriter(w)
|
||||
|
||||
encoder := json.NewEncoder(ww)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
|
||||
|
||||
fmt.Fprintf(ww, "Slug: %s\n", snapshot.Instance.Slug)
|
||||
fmt.Fprintf(ww, "Dest: %s\n", snapshot.Description.Dest)
|
||||
|
||||
fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
|
||||
fmt.Fprintf(ww, "End: %s\n", snapshot.EndTime)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Description =======\n")
|
||||
encoder.Encode(snapshot.Description)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Instance =======\n")
|
||||
encoder.Encode(snapshot.Instance)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Errors =======\n")
|
||||
fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic)
|
||||
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart)
|
||||
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop)
|
||||
|
||||
fmt.Fprintf(ww, "Whitebox: %s\n", snapshot.ErrWhitebox)
|
||||
fmt.Fprintf(ww, "Blackbox: %s\n", snapshot.ErrBlackbox)
|
||||
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Manifest =======\n")
|
||||
for _, file := range snapshot.Manifest {
|
||||
io.WriteString(ww, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
|
||||
|
||||
return ww.Sum()
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this snapshot.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
|
||||
stream.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := env.Create(reportPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = io.WriteString(report, snapshot.String())
|
||||
return err
|
||||
}, stream, "Writing snapshot report")
|
||||
}
|
||||
|
||||
// Strings turns this backup into a string for the BackupReport.
|
||||
func (backup Backup) String() string {
|
||||
var builder strings.Builder
|
||||
backup.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report formats a report for this backup, and writes it into Writer.
|
||||
func (backup Backup) Report(w io.Writer) (int, error) {
|
||||
cw := countwriter.NewCountWriter(w)
|
||||
|
||||
encoder := json.NewEncoder(cw)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(cw, "======= Backup =======\n")
|
||||
|
||||
fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
|
||||
fmt.Fprintf(cw, "End: %s\n", backup.EndTime)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Description =======\n")
|
||||
encoder.Encode(backup.Description)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Errors =======\n")
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Snapshots =======\n")
|
||||
for _, s := range backup.InstanceSnapshots {
|
||||
io.WriteString(cw, s.String())
|
||||
io.WriteString(cw, "\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "======= Manifest =======\n")
|
||||
for _, file := range backup.Manifest {
|
||||
io.WriteString(cw, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
return cw.Sum()
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this backup.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (backup Backup) WriteReport(env environment.Environment, stream stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
|
||||
stream.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := env.Create(reportPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = io.WriteString(report, backup.String())
|
||||
return err
|
||||
}, stream, "Writing backup report")
|
||||
}
|
||||
125
internal/component/snapshots/snapshot.go
Normal file
125
internal/component/snapshots/snapshot.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package snapshots
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
|
||||
"github.com/tkw1536/goprogram/status"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// SnapshotDescription is a description for a snapshot
|
||||
type SnapshotDescription struct {
|
||||
Dest string // destination path
|
||||
Log bool // should we log the creation of this snapshot?
|
||||
Keepalive bool // should we keep the instance alive while making the snapshot?
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Snapshot struct {
|
||||
Description SnapshotDescription
|
||||
Instance models.Instance
|
||||
|
||||
// Start and End Time of the snapshot
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// Generic Panic that may have occured
|
||||
ErrPanic interface{}
|
||||
ErrStart error
|
||||
ErrStop error
|
||||
ErrWhitebox map[string]error
|
||||
ErrBlackbox map[string]error
|
||||
|
||||
// List of files included
|
||||
WithManifest
|
||||
}
|
||||
|
||||
// Snapshot creates a new snapshot of this instance into dest
|
||||
func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
|
||||
// setup the snapshot
|
||||
snapshot.Description = desc
|
||||
snapshot.Instance = instance.Instance
|
||||
|
||||
// capture anything critical, and write the end time
|
||||
defer func() {
|
||||
snapshot.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
snapshot.StartTime = time.Now().UTC()
|
||||
|
||||
snapshot.ErrWhitebox = snapshot.makeParts(io, snapshots, instance, false)
|
||||
snapshot.ErrBlackbox = snapshot.makeParts(io, snapshots, instance, true)
|
||||
|
||||
snapshot.EndTime = time.Now().UTC()
|
||||
return nil
|
||||
}, io, "Writing snapshot files")
|
||||
|
||||
slices.Sort(snapshot.Manifest)
|
||||
return
|
||||
}
|
||||
|
||||
func (snapshot *Snapshot) makeParts(ios stream.IOStream, snapshots *Manager, instance instances.WissKI, needsRunning bool) map[string]error {
|
||||
if !needsRunning && !snapshot.Description.Keepalive {
|
||||
stack := instance.Barrel()
|
||||
|
||||
logging.LogMessage(ios, "Stopping instance")
|
||||
snapshot.ErrStop = stack.Down(ios)
|
||||
|
||||
defer func() {
|
||||
logging.LogMessage(ios, "Starting instance")
|
||||
snapshot.ErrStart = stack.Up(ios)
|
||||
}()
|
||||
}
|
||||
// handle writing the manifest!
|
||||
manifest, done := snapshot.handleManifest(snapshot.Description.Dest)
|
||||
defer done()
|
||||
|
||||
// create a new status
|
||||
st := status.NewWithCompat(ios.Stdout, 0)
|
||||
st.Start()
|
||||
defer st.Stop()
|
||||
|
||||
// get all the components
|
||||
comps := slicesx.FilterClone(snapshots.Snapshotable, func(sc component.Snapshotable) bool {
|
||||
return sc.SnapshotNeedsRunning() == needsRunning
|
||||
})
|
||||
|
||||
results := make(map[string]error, len(comps))
|
||||
|
||||
errors := status.Group[component.Snapshotable, error]{
|
||||
PrefixString: func(item component.Snapshotable, index int) string {
|
||||
return fmt.Sprintf("[snapshot %q]: ", item.Name())
|
||||
},
|
||||
PrefixAlign: true,
|
||||
|
||||
Handler: func(sc component.Snapshotable, index int, writer io.Writer) error {
|
||||
return sc.Snapshot(
|
||||
instance.Instance,
|
||||
component.NewStagingContext(
|
||||
snapshots.Environment,
|
||||
stream.NewIOStream(writer, writer, nil, 0),
|
||||
filepath.Join(snapshot.Description.Dest, sc.SnapshotName()),
|
||||
manifest,
|
||||
),
|
||||
)
|
||||
},
|
||||
|
||||
ResultString: status.DefaultErrorString[component.Snapshotable],
|
||||
}.Use(st, comps)
|
||||
|
||||
for i, wc := range comps {
|
||||
results[wc.Name()] = errors[i]
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
|
@ -1,301 +0,0 @@
|
|||
package snapshots
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
|
||||
"github.com/tkw1536/goprogram/status"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// SnapshotDescription is a description for a snapshot
|
||||
type SnapshotDescription struct {
|
||||
Dest string // destination path
|
||||
Log bool // should we log the creation of this snapshot?
|
||||
Keepalive bool // should we keep the instance alive while making the snapshot?
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Snapshot struct {
|
||||
Description SnapshotDescription
|
||||
Instance models.Instance
|
||||
|
||||
// Start and End Time of the snapshot
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// Generic Panic that may have occured
|
||||
ErrPanic interface{}
|
||||
|
||||
// Errors during starting and stopping the system
|
||||
ErrStart error
|
||||
ErrStop error
|
||||
|
||||
// List of files included
|
||||
Manifest []string
|
||||
|
||||
// Errors during other parts
|
||||
ErrBookkeep error
|
||||
ErrPathbuilder error
|
||||
ErrFilesystem error
|
||||
ErrTriplestore error
|
||||
ErrSQL error
|
||||
}
|
||||
|
||||
func (snapshot Snapshot) String() string {
|
||||
var builder strings.Builder
|
||||
snapshot.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report writes a report from snapshot into w
|
||||
func (snapshot Snapshot) Report(w io.Writer) (int, error) {
|
||||
ww := countwriter.NewCountWriter(w)
|
||||
|
||||
// TODO: Errors of the writer!
|
||||
encoder := json.NewEncoder(ww)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
|
||||
|
||||
fmt.Fprintf(ww, "Slug: %s\n", snapshot.Instance.Slug)
|
||||
fmt.Fprintf(ww, "Dest: %s\n", snapshot.Description.Dest)
|
||||
|
||||
fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
|
||||
fmt.Fprintf(ww, "End: %s\n", snapshot.EndTime)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Description =======\n")
|
||||
encoder.Encode(snapshot.Description)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Instance =======\n")
|
||||
encoder.Encode(snapshot.Instance)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Errors =======\n")
|
||||
fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic)
|
||||
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart)
|
||||
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop)
|
||||
fmt.Fprintf(ww, "Bookkeep: %s\n", snapshot.ErrBookkeep)
|
||||
fmt.Fprintf(ww, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
|
||||
fmt.Fprintf(ww, "Filesystem: %s\n", snapshot.ErrFilesystem)
|
||||
fmt.Fprintf(ww, "Triplestore: %s\n", snapshot.ErrTriplestore)
|
||||
fmt.Fprintf(ww, "SQL: %s\n", snapshot.ErrSQL)
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= Manifest =======\n")
|
||||
for _, file := range snapshot.Manifest {
|
||||
io.WriteString(ww, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(ww, "\n")
|
||||
|
||||
io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
|
||||
|
||||
return ww.Sum()
|
||||
}
|
||||
|
||||
// Snapshot creates a new snapshot of this instance into dest
|
||||
func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
|
||||
// setup the snapshot
|
||||
snapshot.Description = desc
|
||||
snapshot.Instance = instance.Instance
|
||||
|
||||
// capture anything critical, and write the end time
|
||||
defer func() {
|
||||
snapshot.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
snapshot.StartTime = time.Now().UTC()
|
||||
|
||||
snapshot.makeBlackbox(io, snapshots, instance)
|
||||
snapshot.makeWhitebox(io, snapshots, instance)
|
||||
|
||||
snapshot.EndTime = time.Now().UTC()
|
||||
return nil
|
||||
}, io, "Writing snapshot files")
|
||||
|
||||
slices.Sort(snapshot.Manifest)
|
||||
return
|
||||
}
|
||||
|
||||
// makeBlackbox runs the blackbox backup of the system.
|
||||
// It pauses the Instance, if a consistent state is required.
|
||||
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
|
||||
stack := instance.Barrel()
|
||||
|
||||
og := opgroup.NewOpGroup[string](4)
|
||||
|
||||
st := status.NewWithCompat(io.Stdout, 0)
|
||||
st.Start()
|
||||
defer st.Stop()
|
||||
|
||||
// stop the instance (unless it was explicitly asked to not do so!)
|
||||
if !snapshot.Description.Keepalive {
|
||||
logging.LogMessage(io, "Stopping instance")
|
||||
snapshot.ErrStop = stack.Down(io)
|
||||
|
||||
defer func() {
|
||||
logging.LogMessage(io, "Starting instance")
|
||||
snapshot.ErrStart = stack.Up(io)
|
||||
}()
|
||||
}
|
||||
|
||||
// write bookkeeping information
|
||||
og.GoErr(func(files chan<- string) error {
|
||||
line := st.OpenLine("[snapshot bookkeeping]: ", "")
|
||||
defer line.Close()
|
||||
defer fmt.Fprintln(line, "done")
|
||||
|
||||
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
|
||||
fmt.Fprintln(line, bkPath)
|
||||
files <- bkPath
|
||||
|
||||
info, err := snapshots.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer info.Close()
|
||||
|
||||
// print whatever is in the database
|
||||
// TODO: This should be sql code, maybe gorm can do that?
|
||||
_, err = fmt.Fprintf(info, "%#v\n", instance.Instance)
|
||||
return err
|
||||
}, &snapshot.ErrBookkeep)
|
||||
|
||||
// backup the filesystem
|
||||
og.GoErr(func(files chan<- string) error {
|
||||
line := st.OpenLine("[snapshot filesystem]: ", "")
|
||||
defer line.Close()
|
||||
defer fmt.Fprintln(line, "done")
|
||||
|
||||
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
|
||||
|
||||
// copy over whatever is in the base directory
|
||||
defer fmt.Fprintln(line, "done")
|
||||
return fsx.CopyDirectory(snapshots.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
|
||||
fmt.Fprintln(line, dst)
|
||||
files <- dst
|
||||
})
|
||||
|
||||
}, &snapshot.ErrFilesystem)
|
||||
|
||||
// backup the graph db repository
|
||||
og.GoErr(func(files chan<- string) error {
|
||||
line := st.OpenLine("[snapshot triplestore]: ", "")
|
||||
defer line.Close()
|
||||
defer fmt.Fprintln(line, "done")
|
||||
|
||||
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
|
||||
fmt.Fprintln(line, tsPath)
|
||||
files <- tsPath
|
||||
|
||||
nquads, err := snapshots.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer nquads.Close()
|
||||
|
||||
// directly store the result
|
||||
_, err = snapshots.TS.SnapshotDB(nquads, instance.GraphDBRepository)
|
||||
return err
|
||||
}, &snapshot.ErrTriplestore)
|
||||
|
||||
// backup the sql database
|
||||
og.GoErr(func(files chan<- string) error {
|
||||
line := st.OpenLine("[snapshot sql]: ", "")
|
||||
defer line.Close()
|
||||
defer fmt.Fprintln(line, "done")
|
||||
|
||||
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
|
||||
fmt.Fprintln(line, sqlPath)
|
||||
files <- sqlPath
|
||||
|
||||
sql, err := snapshots.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sql.Close()
|
||||
|
||||
// directly store the result
|
||||
return snapshots.SQL.SnapshotDB(io, sql, instance.SqlDatabase)
|
||||
}, &snapshot.ErrSQL)
|
||||
|
||||
// wait for the group!
|
||||
snapshot.waitGroup(io, og)
|
||||
}
|
||||
|
||||
// makeWhitebox runs the whitebox backup of the system.
|
||||
// The instance should be running during this step.
|
||||
func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
|
||||
og := opgroup.NewOpGroup[string](1)
|
||||
|
||||
// write pathbuilders
|
||||
og.GoErr(func(files chan<- string) error {
|
||||
|
||||
pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
|
||||
files <- pbPath
|
||||
|
||||
// create the directory!
|
||||
if err := snapshots.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// put in all the pathbuilders
|
||||
return instance.ExportPathbuilders(pbPath)
|
||||
}, &snapshot.ErrPathbuilder)
|
||||
|
||||
// wait for the group!
|
||||
snapshot.waitGroup(io, og)
|
||||
}
|
||||
|
||||
// waitGroup waits for the
|
||||
func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[string]) {
|
||||
// wait for the messages to return
|
||||
for file := range og.Wait() {
|
||||
// get the relative path to the root of the manifest.
|
||||
// nothing *should* go wrong, but in case it does, use the original path.
|
||||
path, err := filepath.Rel(snapshot.Description.Dest, file)
|
||||
if err != nil {
|
||||
path = file
|
||||
}
|
||||
|
||||
// add the manifest
|
||||
snapshot.Manifest = append(snapshot.Manifest, path)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this snapshot.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
|
||||
stream.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := env.Create(reportPath, environment.DefaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = io.WriteString(report, snapshot.String())
|
||||
return err
|
||||
}, stream, "Writing snapshot report")
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue