301 lines
8.7 KiB
Go
301 lines
8.7 KiB
Go
package snapshots
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
|
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
|
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
|
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
|
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
|
"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
|
|
"github.com/tkw1536/goprogram/status"
|
|
"github.com/tkw1536/goprogram/stream"
|
|
"golang.org/x/exp/slices"
|
|
)
|
|
|
|
// SnapshotDescription is a description for a snapshot
|
|
type SnapshotDescription struct {
|
|
Dest string // destination path
|
|
Log bool // should we log the creation of this snapshot?
|
|
Keepalive bool // should we keep the instance alive while making the snapshot?
|
|
}
|
|
|
|
// Snapshot represents the result of generating a snapshot
|
|
type Snapshot struct {
|
|
Description SnapshotDescription
|
|
Instance models.Instance
|
|
|
|
// Start and End Time of the snapshot
|
|
StartTime time.Time
|
|
EndTime time.Time
|
|
|
|
// Generic Panic that may have occured
|
|
ErrPanic interface{}
|
|
|
|
// Errors during starting and stopping the system
|
|
ErrStart error
|
|
ErrStop error
|
|
|
|
// List of files included
|
|
Manifest []string
|
|
|
|
// Errors during other parts
|
|
ErrBookkeep error
|
|
ErrPathbuilder error
|
|
ErrFilesystem error
|
|
ErrTriplestore error
|
|
ErrSQL error
|
|
}
|
|
|
|
func (snapshot Snapshot) String() string {
|
|
var builder strings.Builder
|
|
snapshot.Report(&builder)
|
|
return builder.String()
|
|
}
|
|
|
|
// Report writes a report from snapshot into w
|
|
func (snapshot Snapshot) Report(w io.Writer) (int, error) {
|
|
ww := countwriter.NewCountWriter(w)
|
|
|
|
// TODO: Errors of the writer!
|
|
encoder := json.NewEncoder(ww)
|
|
encoder.SetIndent("", " ")
|
|
|
|
io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
|
|
|
|
fmt.Fprintf(ww, "Slug: %s\n", snapshot.Instance.Slug)
|
|
fmt.Fprintf(ww, "Dest: %s\n", snapshot.Description.Dest)
|
|
|
|
fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
|
|
fmt.Fprintf(ww, "End: %s\n", snapshot.EndTime)
|
|
io.WriteString(ww, "\n")
|
|
|
|
io.WriteString(ww, "======= Description =======\n")
|
|
encoder.Encode(snapshot.Description)
|
|
io.WriteString(ww, "\n")
|
|
|
|
io.WriteString(ww, "======= Instance =======\n")
|
|
encoder.Encode(snapshot.Instance)
|
|
io.WriteString(ww, "\n")
|
|
|
|
io.WriteString(ww, "======= Errors =======\n")
|
|
fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic)
|
|
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart)
|
|
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop)
|
|
fmt.Fprintf(ww, "Bookkeep: %s\n", snapshot.ErrBookkeep)
|
|
fmt.Fprintf(ww, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
|
|
fmt.Fprintf(ww, "Filesystem: %s\n", snapshot.ErrFilesystem)
|
|
fmt.Fprintf(ww, "Triplestore: %s\n", snapshot.ErrTriplestore)
|
|
fmt.Fprintf(ww, "SQL: %s\n", snapshot.ErrSQL)
|
|
io.WriteString(ww, "\n")
|
|
|
|
io.WriteString(ww, "======= Manifest =======\n")
|
|
for _, file := range snapshot.Manifest {
|
|
io.WriteString(ww, file+"\n")
|
|
}
|
|
|
|
io.WriteString(ww, "\n")
|
|
|
|
io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
|
|
|
|
return ww.Sum()
|
|
}
|
|
|
|
// Snapshot creates a new snapshot of this instance into dest
|
|
func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
|
|
// setup the snapshot
|
|
snapshot.Description = desc
|
|
snapshot.Instance = instance.Instance
|
|
|
|
// capture anything critical, and write the end time
|
|
defer func() {
|
|
snapshot.ErrPanic = recover()
|
|
}()
|
|
|
|
// do the create keeping track of time!
|
|
logging.LogOperation(func() error {
|
|
snapshot.StartTime = time.Now().UTC()
|
|
|
|
snapshot.makeBlackbox(io, snapshots, instance)
|
|
snapshot.makeWhitebox(io, snapshots, instance)
|
|
|
|
snapshot.EndTime = time.Now().UTC()
|
|
return nil
|
|
}, io, "Writing snapshot files")
|
|
|
|
slices.Sort(snapshot.Manifest)
|
|
return
|
|
}
|
|
|
|
// makeBlackbox runs the blackbox backup of the system.
|
|
// It pauses the Instance, if a consistent state is required.
|
|
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
|
|
stack := instance.Barrel()
|
|
|
|
og := opgroup.NewOpGroup[string](4)
|
|
|
|
st := status.NewWithCompat(io.Stdout, 0)
|
|
st.Start()
|
|
defer st.Stop()
|
|
|
|
// stop the instance (unless it was explicitly asked to not do so!)
|
|
if !snapshot.Description.Keepalive {
|
|
logging.LogMessage(io, "Stopping instance")
|
|
snapshot.ErrStop = stack.Down(io)
|
|
|
|
defer func() {
|
|
logging.LogMessage(io, "Starting instance")
|
|
snapshot.ErrStart = stack.Up(io)
|
|
}()
|
|
}
|
|
|
|
// write bookkeeping information
|
|
og.GoErr(func(files chan<- string) error {
|
|
line := st.OpenLine("[snapshot bookkeeping]: ", "")
|
|
defer line.Close()
|
|
defer fmt.Fprintln(line, "done")
|
|
|
|
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
|
|
fmt.Fprintln(line, bkPath)
|
|
files <- bkPath
|
|
|
|
info, err := snapshots.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer info.Close()
|
|
|
|
// print whatever is in the database
|
|
// TODO: This should be sql code, maybe gorm can do that?
|
|
_, err = fmt.Fprintf(info, "%#v\n", instance.Instance)
|
|
return err
|
|
}, &snapshot.ErrBookkeep)
|
|
|
|
// backup the filesystem
|
|
og.GoErr(func(files chan<- string) error {
|
|
line := st.OpenLine("[snapshot filesystem]: ", "")
|
|
defer line.Close()
|
|
defer fmt.Fprintln(line, "done")
|
|
|
|
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
|
|
|
|
// copy over whatever is in the base directory
|
|
defer fmt.Fprintln(line, "done")
|
|
return fsx.CopyDirectory(snapshots.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
|
|
fmt.Fprintln(line, dst)
|
|
files <- dst
|
|
})
|
|
|
|
}, &snapshot.ErrFilesystem)
|
|
|
|
// backup the graph db repository
|
|
og.GoErr(func(files chan<- string) error {
|
|
line := st.OpenLine("[snapshot triplestore]: ", "")
|
|
defer line.Close()
|
|
defer fmt.Fprintln(line, "done")
|
|
|
|
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
|
|
fmt.Fprintln(line, tsPath)
|
|
files <- tsPath
|
|
|
|
nquads, err := snapshots.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer nquads.Close()
|
|
|
|
// directly store the result
|
|
_, err = snapshots.TS.SnapshotDB(nquads, instance.GraphDBRepository)
|
|
return err
|
|
}, &snapshot.ErrTriplestore)
|
|
|
|
// backup the sql database
|
|
og.GoErr(func(files chan<- string) error {
|
|
line := st.OpenLine("[snapshot sql]: ", "")
|
|
defer line.Close()
|
|
defer fmt.Fprintln(line, "done")
|
|
|
|
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
|
|
fmt.Fprintln(line, sqlPath)
|
|
files <- sqlPath
|
|
|
|
sql, err := snapshots.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer sql.Close()
|
|
|
|
// directly store the result
|
|
return snapshots.SQL.SnapshotDB(io, sql, instance.SqlDatabase)
|
|
}, &snapshot.ErrSQL)
|
|
|
|
// wait for the group!
|
|
snapshot.waitGroup(io, og)
|
|
}
|
|
|
|
// makeWhitebox runs the whitebox backup of the system.
|
|
// The instance should be running during this step.
|
|
func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
|
|
og := opgroup.NewOpGroup[string](1)
|
|
|
|
// write pathbuilders
|
|
og.GoErr(func(files chan<- string) error {
|
|
|
|
pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
|
|
files <- pbPath
|
|
|
|
// create the directory!
|
|
if err := snapshots.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
|
|
return err
|
|
}
|
|
|
|
// put in all the pathbuilders
|
|
return instance.ExportPathbuilders(pbPath)
|
|
}, &snapshot.ErrPathbuilder)
|
|
|
|
// wait for the group!
|
|
snapshot.waitGroup(io, og)
|
|
}
|
|
|
|
// waitGroup waits for the
|
|
func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[string]) {
|
|
// wait for the messages to return
|
|
for file := range og.Wait() {
|
|
// get the relative path to the root of the manifest.
|
|
// nothing *should* go wrong, but in case it does, use the original path.
|
|
path, err := filepath.Rel(snapshot.Description.Dest, file)
|
|
if err != nil {
|
|
path = file
|
|
}
|
|
|
|
// add the manifest
|
|
snapshot.Manifest = append(snapshot.Manifest, path)
|
|
}
|
|
}
|
|
|
|
// WriteReport writes out the report belonging to this snapshot.
|
|
// It is a separate function, to allow writing it indepenently of the rest.
|
|
func (snapshot *Snapshot) WriteReport(env environment.Environment, stream stream.IOStream) error {
|
|
return logging.LogOperation(func() error {
|
|
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
|
|
stream.Println(reportPath)
|
|
|
|
// create the report file!
|
|
report, err := env.Create(reportPath, environment.DefaultFilePerm)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer report.Close()
|
|
|
|
// print the report into it!
|
|
_, err = io.WriteString(report, snapshot.String())
|
|
return err
|
|
}, stream, "Writing snapshot report")
|
|
}
|