rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

{snapshot,backup}: Write machine-readable report

Browse source
This commit is contained in:
Tom Wiesing 2023-03-19 17:38:36 +01:00
parent c79dcc6b90
commit b6d3575ee9
No known key found for this signature in database
6 changed files with 196 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

14
cmd/snapshot.go
View file

@ -14,6 +14,9 @@ type snapshot struct {
Keepalive bool `short:"k" long:"keepalive" description:"keep instance running while taking a backup. might lead to inconsistent state"`
StagingOnly bool `short:"s" long:"staging-only" description:"do not package into a snapshot archive, but only create a staging directory"`
Parts []string `short:"p" long:"parts" description:"parts to include in snapshots. defaults to all parts, use l to list all available parts"`
List bool `short:"l" long:"list-parts" description:"list available parts"`
Positionals struct {
Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"`
Dest string "positional-arg-name:\"DEST\" description:\"destination path to write snapshot archive to. defaults to the `snapshots/archives/` directory\""
@ -43,6 +46,14 @@ var errSnapshotWissKI = exit.Error{
func (sn snapshot) Run(context wisski_distillery.Context) error {
dis := context.Environment
// list available parts
if sn.List {
for _, part := range dis.Exporter().Parts() {
context.Println(part)
}
return nil
}
// find the instance!
instance, err := dis.Instances().WissKI(context.Context, sn.Positionals.Slug)
if err != nil {
@ -54,6 +65,9 @@ func (sn snapshot) Run(context wisski_distillery.Context) error {
Dest: sn.Positionals.Dest,
StagingOnly: sn.StagingOnly,
SnapshotDescription: exporter.SnapshotDescription{
Parts: sn.Parts,
},
Instance: instance,
})

2
go.mod
View file

@ -16,7 +16,7 @@ require (
github.com/pquerna/otp v1.4.0
github.com/rs/zerolog v1.29.0
github.com/tkw1536/goprogram v0.3.5
github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194
github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874
github.com/yuin/goldmark v1.5.4
github.com/yuin/goldmark-meta v1.1.0
golang.org/x/crypto v0.7.0

4
go.sum
View file

@ -115,8 +115,8 @@ github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtp
github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM=
github.com/tkw1536/goprogram v0.3.5 h1:S0axKo3R/vGa4zhYqYDKAZEPhAfwUSSeMtVwnAu4sNY=
github.com/tkw1536/goprogram v0.3.5/go.mod h1:pYr4dMHOSVurbPQ4KTR0ett8XWNISbsRS6zlh9Nsxa8=
github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194 h1:IRgZUQaQc/p2yTPT1eUL/aeib3POEClM4UqejtOGA3M=
github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew=
github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874 h1:BrsHJnkecpO1OOXiuFcJG2t4EiaLhhMdLMrfPldYWl4=
github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

51
internal/dis/component/exporter/iface.go
View file

@ -9,11 +9,13 @@ import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
"github.com/tkw1536/pkglib/collection"
"github.com/tkw1536/pkglib/status"
)
@ -28,6 +30,11 @@ type ExportTask struct {
// To generated an unpacked directory, set [StagingOnly] to true.
StagingOnly bool
// Parts explicitly lists parts to include inside the snapshot.
// If non-empty, only include parts with the specified names.
// if empty, include all possible components.
Parts []string
// Instance is the instance to generate a snapshot of.
// To generate a backup, leave this to be nil.
Instance *wisski.WissKI
@ -41,12 +48,26 @@ type ExportTask struct {
// export is implemented by [Backup] and [Snapshot]
type export interface {
LogEntry() models.Export
Report(w io.Writer) (int, error)
// ReportPlain writes a plaintext report summary into w
ReportPlain(w io.Writer) error
// ReportMachine writes a machine readable report summary into w
ReportMachine(w io.Writer) error
}
// Parts lists all available snapshot parts
func (exporter *Exporter) Parts() []string {
return collection.MapSlice(exporter.Dependencies.Snapshotable, func(c component.Snapshotable) string { return c.SnapshotName() })
}
const (
ReportPlainPath = "README.txt"
ReportMachinePath = "report.json"
)
// MakeExport performs an export task as described by flags.
// Output is directed to the provided io.
func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, task ExportTask) (err error) {
// extract parameters
Title := "Backup"
Slug := ""
@ -110,21 +131,37 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
// create a log entry
entry = sl.LogEntry()
// find the report path
reportPath := filepath.Join(stagingDir, "report.txt")
// write the machine report
{
reportPath := filepath.Join(stagingDir, ReportMachinePath)
fmt.Fprintln(progress, reportPath)
// create the path
report, err := fsx.Create(reportPath, fsx.DefaultFilePerm)
if err != nil {
return err
}
// and write out the report
{
_, err := sl.Report(report)
if err := sl.ReportMachine(report); err != nil {
return err
}
}
// write the plaintext report
{
reportPath := filepath.Join(stagingDir, ReportPlainPath)
fmt.Fprintln(progress, reportPath)
report, err := fsx.Create(reportPath, fsx.DefaultFilePerm)
if err != nil {
return err
}
if err := sl.ReportPlain(report); err != nil {
return err
}
}
return nil
}, progress, "Generating %s", Title)
// if we only requested staging

26
internal/dis/component/exporter/report.go
View file

@ -13,12 +13,15 @@ func (snapshot Snapshot) String() string {
builder := pools.GetBuilder()
defer pools.ReleaseBuilder(builder)
snapshot.Report(builder)
snapshot.ReportPlain(builder)
return builder.String()
}
// Report writes a report from snapshot into w
func (snapshot Snapshot) Report(w io.Writer) (int, error) {
func (snapshot Snapshot) ReportMachine(w io.Writer) error {
return json.NewEncoder(w).Encode(snapshot)
}
func (snapshot Snapshot) ReportPlain(w io.Writer) error {
ww := &sequence.Writer{Writer: w}
encoder := json.NewEncoder(ww)
@ -46,8 +49,7 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) {
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart)
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop)
fmt.Fprintf(ww, "Whitebox: %s\n", snapshot.ErrWhitebox)
fmt.Fprintf(ww, "Blackbox: %s\n", snapshot.ErrBlackbox)
fmt.Fprintf(ww, "Errors: %s\n", snapshot.Errors)
io.WriteString(ww, "\n")
@ -60,7 +62,8 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) {
io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
return ww.Sum()
_, err := ww.Sum()
return err
}
// Strings turns this backup into a string for the BackupReport.
@ -68,12 +71,16 @@ func (backup Backup) String() string {
builder := pools.GetBuilder()
defer pools.ReleaseBuilder(builder)
backup.Report(builder)
backup.ReportPlain(builder)
return builder.String()
}
func (backup Backup) ReportMachine(w io.Writer) error {
return json.NewEncoder(w).Encode(backup)
}
// Report formats a report for this backup, and writes it into Writer.
func (backup Backup) Report(w io.Writer) (int, error) {
func (backup Backup) ReportPlain(w io.Writer) error {
cw := &sequence.Writer{Writer: w}
encoder := json.NewEncoder(cw)
@ -110,5 +117,6 @@ func (backup Backup) Report(w io.Writer) (int, error) {
io.WriteString(cw, "\n")
return cw.Sum()
_, err := cw.Sum()
return err
}

123
internal/dis/component/exporter/snapshot.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"time"
@ -12,8 +13,11 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/locker"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/rs/zerolog"
"github.com/tkw1536/pkglib/collection"
"github.com/tkw1536/pkglib/contextx"
"github.com/tkw1536/pkglib/status"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
@ -21,6 +25,8 @@ import (
type SnapshotDescription struct {
Dest string // destination path
Keepalive bool // should we keep the instance alive while making the snapshot?
Parts []string // SnapshotName()s of the components to include.
}
// Snapshot represents the result of generating a snapshot
@ -36,15 +42,23 @@ type Snapshot struct {
ErrPanic interface{}
ErrStart error
ErrStop error
ErrWhitebox map[string]error
ErrBlackbox map[string]error
// Errors holds errors for each component
Errors map[string]error
// Logs contains logfiles for each component
Logs map[string]string
// List of files included
WithManifest
// snapshotables that are running and not running
partsRunning []component.Snapshotable `json:"-"`
partsStopped []component.Snapshotable `json:"-"`
}
// Snapshot creates a new snapshot of this instance into dest
func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) {
func (exporter *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) {
logging.LogMessage(progress, "Locking instance")
if !instance.Locker().TryLock(ctx) {
@ -58,11 +72,16 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
}
defer func() {
logging.LogMessage(progress, "Unlocking instance")
ctx, cancel := contextx.Anyways(ctx, time.Second)
defer cancel()
instance.Locker().Unlock(ctx)
}()
// setup the snapshot
snapshot.Description = desc
exporter.resolveParts(ctx, desc.Parts, &snapshot)
snapshot.Instance = instance.Instance
// capture anything critical, and write the end time
@ -74,10 +93,15 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
logging.LogOperation(func() error {
snapshot.StartTime = time.Now().UTC()
snapshot.ErrWhitebox = snapshot.makeParts(ctx, progress, snapshots, instance, false)
snapshot.ErrBlackbox = snapshot.makeParts(ctx, progress, snapshots, instance, true)
wboxerr, wboxmsg := snapshot.makeParts(ctx, progress, exporter, instance, false)
bboxerr, bboxlog := snapshot.makeParts(ctx, progress, exporter, instance, true)
snapshot.EndTime = time.Now().UTC()
// collection all the errors and logs
snapshot.Errors = collection.Append(wboxerr, bboxerr)
snapshot.Logs = collection.Append(wboxmsg, bboxlog)
return nil
}, progress, "Writing snapshot files")
@ -85,7 +109,53 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
return
}
func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) map[string]error {
// resolveParts resolves parts, and writes it into snapshot.Description.Parts.
// Also sets up snapshot.partsRunning and snapshot.partsStopped.
// sends a warning about unknown parts into the logger in context.
func (snapshots *Exporter) resolveParts(ctx context.Context, parts []string, snapshot *Snapshot) {
partMap := make(map[string]component.Snapshotable, len(snapshots.Dependencies.Snapshotable))
for _, part := range snapshots.Dependencies.Snapshotable {
partMap[part.SnapshotName()] = part
}
// filter the parts (if requested)
if len(parts) != 0 {
keys := make(map[string]struct{}, len(parts))
for _, part := range parts {
keys[part] = struct{}{}
}
// delete all the parts which weren't explicitly requested
for part := range partMap {
if _, ok := keys[part]; !ok {
delete(partMap, part)
} else {
delete(keys, part)
}
}
// throw a warning for unknown parts
for key := range keys {
zerolog.Ctx(ctx).Warn().Str("part", key).Msg("ignoring unknown snapshot part")
}
}
// sort the names of all requested parts
snapshot.Description.Parts = maps.Keys(partMap)
slices.Sort(snapshot.Description.Parts)
// and setup the map for running and stopped parts!
for _, name := range snapshot.Description.Parts {
part := partMap[name]
if part.SnapshotNeedsRunning() {
snapshot.partsRunning = append(snapshot.partsRunning, part)
} else {
snapshot.partsStopped = append(snapshot.partsStopped, part)
}
}
}
func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) (errmap map[string]error, logmap map[string]string) {
if !needsRunning && !snapshot.Description.Keepalive {
stack := instance.Barrel().Stack()
@ -106,14 +176,16 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna
st.Start()
defer st.Stop()
// get all the components
comps := collection.FilterClone(snapshots.Dependencies.Snapshotable, func(sc component.Snapshotable) bool {
return sc.SnapshotNeedsRunning() == needsRunning
})
// get the components
var comps []component.Snapshotable
if needsRunning {
comps = snapshot.partsRunning
} else {
comps = snapshot.partsStopped
}
results := make(map[string]error, len(comps))
errors, _ := status.Group[component.Snapshotable, error]{
// run each of the parts
errors, ids := status.Group[component.Snapshotable, error]{
PrefixString: func(item component.Snapshotable, index int) string {
return fmt.Sprintf("[snapshot %q]: ", item.Name())
},
@ -134,8 +206,29 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna
ResultString: status.DefaultErrorString[component.Snapshotable],
}.Use(st, comps)
// keep all the log files
files := st.Keep()
// store errors and logs
errmap = make(map[string]error, len(comps))
logmap = make(map[string]string, len(comps))
for i, wc := range comps {
results[wc.Name()] = errors[i]
name := wc.SnapshotName()
errmap[name] = errors[i]
// read the logfile
logfile := files[ids[i]]
bytes, err := os.ReadFile(logfile)
if err != nil {
zerolog.Ctx(ctx).Err(err).Str("component", name).Msg("unable to copy logfile")
continue
}
return results
// delete it, but store the content in the results
os.Remove(logfile)
logmap[name] = string(bytes)
}
return
}