{snapshot,backup}: Write machine-readable report

cmd/snapshot.go

@@ -14,6 +14,9 @@ type snapshot struct {
 	Keepalive   bool `short:"k" long:"keepalive" description:"keep instance running while taking a backup. might lead to inconsistent state"`
 	StagingOnly bool `short:"s" long:"staging-only" description:"do not package into a snapshot archive, but only create a staging directory"`
 
+	Parts []string `short:"p" long:"parts" description:"parts to include in snapshots. defaults to all parts, use l to list all available parts"`
+	List  bool     `short:"l" long:"list-parts" description:"list available parts"`
+
 	Positionals struct {
 		Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"`
 		Dest string "positional-arg-name:\"DEST\" description:\"destination path to write snapshot archive to. defaults to the `snapshots/archives/` directory\""
@@ -43,6 +46,14 @@ var errSnapshotWissKI = exit.Error{
 func (sn snapshot) Run(context wisski_distillery.Context) error {
 	dis := context.Environment
 
+	// list available parts
+	if sn.List {
+		for _, part := range dis.Exporter().Parts() {
+			context.Println(part)
+		}
+		return nil
+	}
+
 	// find the instance!
 	instance, err := dis.Instances().WissKI(context.Context, sn.Positionals.Slug)
 	if err != nil {
@@ -54,6 +65,9 @@ func (sn snapshot) Run(context wisski_distillery.Context) error {
 		Dest:        sn.Positionals.Dest,
 		StagingOnly: sn.StagingOnly,
 
+		SnapshotDescription: exporter.SnapshotDescription{
+			Parts: sn.Parts,
+		},
 		Instance: instance,
 	})
 

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/pquerna/otp v1.4.0
 	github.com/rs/zerolog v1.29.0
 	github.com/tkw1536/goprogram v0.3.5
-	github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194
+	github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874
 	github.com/yuin/goldmark v1.5.4
 	github.com/yuin/goldmark-meta v1.1.0
 	golang.org/x/crypto v0.7.0

go.sum

@@ -115,8 +115,8 @@ github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtp
 github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM=
 github.com/tkw1536/goprogram v0.3.5 h1:S0axKo3R/vGa4zhYqYDKAZEPhAfwUSSeMtVwnAu4sNY=
 github.com/tkw1536/goprogram v0.3.5/go.mod h1:pYr4dMHOSVurbPQ4KTR0ett8XWNISbsRS6zlh9Nsxa8=
-github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194 h1:IRgZUQaQc/p2yTPT1eUL/aeib3POEClM4UqejtOGA3M=
+github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874 h1:BrsHJnkecpO1OOXiuFcJG2t4EiaLhhMdLMrfPldYWl4=
-github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew=
+github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

internal/dis/component/exporter/iface.go

@@ -9,11 +9,13 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
 	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/FAU-CDI/wisski-distillery/internal/wisski"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/targz"
+	"github.com/tkw1536/pkglib/collection"
 	"github.com/tkw1536/pkglib/status"
 )
 
@@ -28,6 +30,11 @@ type ExportTask struct {
 	// To generated an unpacked directory, set [StagingOnly] to true.
 	StagingOnly bool
 
+	// Parts explicitly lists parts to include inside the snapshot.
+	// If non-empty, only include parts with the specified names.
+	// if empty, include all possible components.
+	Parts []string
+
 	// Instance is the instance to generate a snapshot of.
 	// To generate a backup, leave this to be nil.
 	Instance *wisski.WissKI
@@ -41,12 +48,26 @@ type ExportTask struct {
 // export is implemented by [Backup] and [Snapshot]
 type export interface {
 	LogEntry() models.Export
-	Report(w io.Writer) (int, error)
+	// ReportPlain writes a plaintext report summary into w
+	ReportPlain(w io.Writer) error
+	// ReportMachine writes a machine readable report summary into w
+	ReportMachine(w io.Writer) error
 }
 
+// Parts lists all available snapshot parts
+func (exporter *Exporter) Parts() []string {
+	return collection.MapSlice(exporter.Dependencies.Snapshotable, func(c component.Snapshotable) string { return c.SnapshotName() })
+}
+
+const (
+	ReportPlainPath   = "README.txt"
+	ReportMachinePath = "report.json"
+)
+
 // MakeExport performs an export task as described by flags.
 // Output is directed to the provided io.
 func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, task ExportTask) (err error) {
+
 	// extract parameters
 	Title := "Backup"
 	Slug := ""
@@ -110,21 +131,37 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
 		// create a log entry
 		entry = sl.LogEntry()
 
-		// find the report path
+		// write the machine report
-		reportPath := filepath.Join(stagingDir, "report.txt")
-		fmt.Fprintln(progress, reportPath)
-
-		// create the path
-		report, err := fsx.Create(reportPath, fsx.DefaultFilePerm)
-		if err != nil {
-			return err
-		}
-
-		// and write out the report
 		{
-			_, err := sl.Report(report)
+			reportPath := filepath.Join(stagingDir, ReportMachinePath)
-			return err
+			fmt.Fprintln(progress, reportPath)
+
+			report, err := fsx.Create(reportPath, fsx.DefaultFilePerm)
+			if err != nil {
+				return err
+			}
+
+			if err := sl.ReportMachine(report); err != nil {
+				return err
+			}
 		}
+
+		// write the plaintext report
+		{
+			reportPath := filepath.Join(stagingDir, ReportPlainPath)
+			fmt.Fprintln(progress, reportPath)
+
+			report, err := fsx.Create(reportPath, fsx.DefaultFilePerm)
+			if err != nil {
+				return err
+			}
+
+			if err := sl.ReportPlain(report); err != nil {
+				return err
+			}
+		}
+
+		return nil
 	}, progress, "Generating %s", Title)
 
 	// if we only requested staging

internal/dis/component/exporter/report.go

@@ -13,12 +13,15 @@ func (snapshot Snapshot) String() string {
 	builder := pools.GetBuilder()
 	defer pools.ReleaseBuilder(builder)
 
-	snapshot.Report(builder)
+	snapshot.ReportPlain(builder)
 	return builder.String()
 }
 
-// Report writes a report from snapshot into w
+func (snapshot Snapshot) ReportMachine(w io.Writer) error {
-func (snapshot Snapshot) Report(w io.Writer) (int, error) {
+	return json.NewEncoder(w).Encode(snapshot)
+}
+
+func (snapshot Snapshot) ReportPlain(w io.Writer) error {
 	ww := &sequence.Writer{Writer: w}
 
 	encoder := json.NewEncoder(ww)
@@ -46,8 +49,7 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) {
 	fmt.Fprintf(ww, "Start:       %s\n", snapshot.ErrStart)
 	fmt.Fprintf(ww, "Stop:        %s\n", snapshot.ErrStop)
 
-	fmt.Fprintf(ww, "Whitebox:    %s\n", snapshot.ErrWhitebox)
+	fmt.Fprintf(ww, "Errors:    %s\n", snapshot.Errors)
-	fmt.Fprintf(ww, "Blackbox:    %s\n", snapshot.ErrBlackbox)
 
 	io.WriteString(ww, "\n")
 
@@ -60,7 +62,8 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) {
 
 	io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
 
-	return ww.Sum()
+	_, err := ww.Sum()
+	return err
 }
 
 // Strings turns this backup into a string for the BackupReport.
@@ -68,12 +71,16 @@ func (backup Backup) String() string {
 	builder := pools.GetBuilder()
 	defer pools.ReleaseBuilder(builder)
 
-	backup.Report(builder)
+	backup.ReportPlain(builder)
 	return builder.String()
 }
 
+func (backup Backup) ReportMachine(w io.Writer) error {
+	return json.NewEncoder(w).Encode(backup)
+}
+
 // Report formats a report for this backup, and writes it into Writer.
-func (backup Backup) Report(w io.Writer) (int, error) {
+func (backup Backup) ReportPlain(w io.Writer) error {
 	cw := &sequence.Writer{Writer: w}
 
 	encoder := json.NewEncoder(cw)
@@ -110,5 +117,6 @@ func (backup Backup) Report(w io.Writer) (int, error) {
 
 	io.WriteString(cw, "\n")
 
-	return cw.Sum()
+	_, err := cw.Sum()
+	return err
 }

internal/dis/component/exporter/snapshot.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -12,8 +13,11 @@ import (
 	"github.com/FAU-CDI/wisski-distillery/internal/wisski"
 	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/locker"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/rs/zerolog"
 	"github.com/tkw1536/pkglib/collection"
+	"github.com/tkw1536/pkglib/contextx"
 	"github.com/tkw1536/pkglib/status"
+	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 )
 
@@ -21,6 +25,8 @@ import (
 type SnapshotDescription struct {
 	Dest      string // destination path
 	Keepalive bool   // should we keep the instance alive while making the snapshot?
+
+	Parts []string // SnapshotName()s of the components to include.
 }
 
 // Snapshot represents the result of generating a snapshot
@@ -33,18 +39,26 @@ type Snapshot struct {
 	EndTime   time.Time
 
 	// Generic Panic that may have occured
-	ErrPanic    interface{}
+	ErrPanic interface{}
-	ErrStart    error
+	ErrStart error
-	ErrStop     error
+	ErrStop  error
-	ErrWhitebox map[string]error
+
-	ErrBlackbox map[string]error
+	// Errors holds errors for each component
+	Errors map[string]error
+
+	// Logs contains logfiles for each component
+	Logs map[string]string
 
 	// List of files included
 	WithManifest
+
+	// snapshotables that are running and not running
+	partsRunning []component.Snapshotable `json:"-"`
+	partsStopped []component.Snapshotable `json:"-"`
 }
 
 // Snapshot creates a new snapshot of this instance into dest
-func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) {
+func (exporter *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) {
 
 	logging.LogMessage(progress, "Locking instance")
 	if !instance.Locker().TryLock(ctx) {
@@ -58,11 +72,16 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
 	}
 	defer func() {
 		logging.LogMessage(progress, "Unlocking instance")
+
+		ctx, cancel := contextx.Anyways(ctx, time.Second)
+		defer cancel()
+
 		instance.Locker().Unlock(ctx)
 	}()
 
 	// setup the snapshot
 	snapshot.Description = desc
+	exporter.resolveParts(ctx, desc.Parts, &snapshot)
 	snapshot.Instance = instance.Instance
 
 	// capture anything critical, and write the end time
@@ -74,10 +93,15 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
 	logging.LogOperation(func() error {
 		snapshot.StartTime = time.Now().UTC()
 
-		snapshot.ErrWhitebox = snapshot.makeParts(ctx, progress, snapshots, instance, false)
+		wboxerr, wboxmsg := snapshot.makeParts(ctx, progress, exporter, instance, false)
-		snapshot.ErrBlackbox = snapshot.makeParts(ctx, progress, snapshots, instance, true)
+		bboxerr, bboxlog := snapshot.makeParts(ctx, progress, exporter, instance, true)
 
 		snapshot.EndTime = time.Now().UTC()
+
+		// collection all the errors and logs
+		snapshot.Errors = collection.Append(wboxerr, bboxerr)
+		snapshot.Logs = collection.Append(wboxmsg, bboxlog)
+
 		return nil
 	}, progress, "Writing snapshot files")
 
@@ -85,7 +109,53 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
 	return
 }
 
-func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) map[string]error {
+// resolveParts resolves parts, and writes it into snapshot.Description.Parts.
+// Also sets up snapshot.partsRunning and snapshot.partsStopped.
+// sends a warning about unknown parts into the logger in context.
+func (snapshots *Exporter) resolveParts(ctx context.Context, parts []string, snapshot *Snapshot) {
+	partMap := make(map[string]component.Snapshotable, len(snapshots.Dependencies.Snapshotable))
+	for _, part := range snapshots.Dependencies.Snapshotable {
+		partMap[part.SnapshotName()] = part
+	}
+
+	// filter the parts (if requested)
+	if len(parts) != 0 {
+		keys := make(map[string]struct{}, len(parts))
+		for _, part := range parts {
+			keys[part] = struct{}{}
+		}
+
+		// delete all the parts which weren't explicitly requested
+		for part := range partMap {
+			if _, ok := keys[part]; !ok {
+				delete(partMap, part)
+			} else {
+				delete(keys, part)
+			}
+		}
+
+		// throw a warning for unknown parts
+		for key := range keys {
+			zerolog.Ctx(ctx).Warn().Str("part", key).Msg("ignoring unknown snapshot part")
+		}
+	}
+
+	// sort the names of all requested parts
+	snapshot.Description.Parts = maps.Keys(partMap)
+	slices.Sort(snapshot.Description.Parts)
+
+	// and setup the map for running and stopped parts!
+	for _, name := range snapshot.Description.Parts {
+		part := partMap[name]
+		if part.SnapshotNeedsRunning() {
+			snapshot.partsRunning = append(snapshot.partsRunning, part)
+		} else {
+			snapshot.partsStopped = append(snapshot.partsStopped, part)
+		}
+	}
+}
+
+func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) (errmap map[string]error, logmap map[string]string) {
 	if !needsRunning && !snapshot.Description.Keepalive {
 		stack := instance.Barrel().Stack()
 
@@ -106,14 +176,16 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna
 	st.Start()
 	defer st.Stop()
 
-	// get all the components
+	// get the components
-	comps := collection.FilterClone(snapshots.Dependencies.Snapshotable, func(sc component.Snapshotable) bool {
+	var comps []component.Snapshotable
-		return sc.SnapshotNeedsRunning() == needsRunning
+	if needsRunning {
-	})
+		comps = snapshot.partsRunning
+	} else {
+		comps = snapshot.partsStopped
+	}
 
-	results := make(map[string]error, len(comps))
+	// run each of the parts
-
+	errors, ids := status.Group[component.Snapshotable, error]{
-	errors, _ := status.Group[component.Snapshotable, error]{
 		PrefixString: func(item component.Snapshotable, index int) string {
 			return fmt.Sprintf("[snapshot %q]: ", item.Name())
 		},
@@ -134,8 +206,29 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna
 		ResultString: status.DefaultErrorString[component.Snapshotable],
 	}.Use(st, comps)
 
+	// keep all the log files
+	files := st.Keep()
+
+	// store errors and logs
+	errmap = make(map[string]error, len(comps))
+	logmap = make(map[string]string, len(comps))
+
 	for i, wc := range comps {
-		results[wc.Name()] = errors[i]
+		name := wc.SnapshotName()
+		errmap[name] = errors[i]
+
+		// read the logfile
+		logfile := files[ids[i]]
+		bytes, err := os.ReadFile(logfile)
+		if err != nil {
+			zerolog.Ctx(ctx).Err(err).Str("component", name).Msg("unable to copy logfile")
+			continue
+		}
+
+		// delete it, but store the content in the results
+		os.Remove(logfile)
+		logmap[name] = string(bytes)
 	}
-	return results
+
+	return
 }