diff --git a/cmd/snapshot.go b/cmd/snapshot.go index 0300f53..68dfc8b 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -14,6 +14,9 @@ type snapshot struct { Keepalive bool `short:"k" long:"keepalive" description:"keep instance running while taking a backup. might lead to inconsistent state"` StagingOnly bool `short:"s" long:"staging-only" description:"do not package into a snapshot archive, but only create a staging directory"` + Parts []string `short:"p" long:"parts" description:"parts to include in snapshots. defaults to all parts, use l to list all available parts"` + List bool `short:"l" long:"list-parts" description:"list available parts"` + Positionals struct { Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to take a snapshot of"` Dest string "positional-arg-name:\"DEST\" description:\"destination path to write snapshot archive to. defaults to the `snapshots/archives/` directory\"" @@ -43,6 +46,14 @@ var errSnapshotWissKI = exit.Error{ func (sn snapshot) Run(context wisski_distillery.Context) error { dis := context.Environment + // list available parts + if sn.List { + for _, part := range dis.Exporter().Parts() { + context.Println(part) + } + return nil + } + // find the instance! instance, err := dis.Instances().WissKI(context.Context, sn.Positionals.Slug) if err != nil { @@ -54,6 +65,9 @@ func (sn snapshot) Run(context wisski_distillery.Context) error { Dest: sn.Positionals.Dest, StagingOnly: sn.StagingOnly, + SnapshotDescription: exporter.SnapshotDescription{ + Parts: sn.Parts, + }, Instance: instance, }) diff --git a/go.mod b/go.mod index b24ef0c..273ba7f 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/pquerna/otp v1.4.0 github.com/rs/zerolog v1.29.0 github.com/tkw1536/goprogram v0.3.5 - github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194 + github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874 github.com/yuin/goldmark v1.5.4 github.com/yuin/goldmark-meta v1.1.0 golang.org/x/crypto v0.7.0 diff --git a/go.sum b/go.sum index 0a17790..b8e5073 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtp github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM= github.com/tkw1536/goprogram v0.3.5 h1:S0axKo3R/vGa4zhYqYDKAZEPhAfwUSSeMtVwnAu4sNY= github.com/tkw1536/goprogram v0.3.5/go.mod h1:pYr4dMHOSVurbPQ4KTR0ett8XWNISbsRS6zlh9Nsxa8= -github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194 h1:IRgZUQaQc/p2yTPT1eUL/aeib3POEClM4UqejtOGA3M= -github.com/tkw1536/pkglib v0.0.0-20230316111730-af32f59c9194/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew= +github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874 h1:BrsHJnkecpO1OOXiuFcJG2t4EiaLhhMdLMrfPldYWl4= +github.com/tkw1536/pkglib v0.0.0-20230319133918-3edc36187874/go.mod h1:RjPEyRcq+g1GMd3D/o7d9WCtVNXY4QZyFRs9hLlZbew= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/internal/dis/component/exporter/iface.go b/internal/dis/component/exporter/iface.go index 11dfcb0..36e5765 100644 --- a/internal/dis/component/exporter/iface.go +++ b/internal/dis/component/exporter/iface.go @@ -9,11 +9,13 @@ import ( "os" "path/filepath" + "github.com/FAU-CDI/wisski-distillery/internal/dis/component" "github.com/FAU-CDI/wisski-distillery/internal/models" "github.com/FAU-CDI/wisski-distillery/internal/wisski" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/targz" + "github.com/tkw1536/pkglib/collection" "github.com/tkw1536/pkglib/status" ) @@ -28,6 +30,11 @@ type ExportTask struct { // To generated an unpacked directory, set [StagingOnly] to true. StagingOnly bool + // Parts explicitly lists parts to include inside the snapshot. + // If non-empty, only include parts with the specified names. + // if empty, include all possible components. + Parts []string + // Instance is the instance to generate a snapshot of. // To generate a backup, leave this to be nil. Instance *wisski.WissKI @@ -41,12 +48,26 @@ type ExportTask struct { // export is implemented by [Backup] and [Snapshot] type export interface { LogEntry() models.Export - Report(w io.Writer) (int, error) + // ReportPlain writes a plaintext report summary into w + ReportPlain(w io.Writer) error + // ReportMachine writes a machine readable report summary into w + ReportMachine(w io.Writer) error } +// Parts lists all available snapshot parts +func (exporter *Exporter) Parts() []string { + return collection.MapSlice(exporter.Dependencies.Snapshotable, func(c component.Snapshotable) string { return c.SnapshotName() }) +} + +const ( + ReportPlainPath = "README.txt" + ReportMachinePath = "report.json" +) + // MakeExport performs an export task as described by flags. // Output is directed to the provided io. func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, task ExportTask) (err error) { + // extract parameters Title := "Backup" Slug := "" @@ -110,21 +131,37 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta // create a log entry entry = sl.LogEntry() - // find the report path - reportPath := filepath.Join(stagingDir, "report.txt") - fmt.Fprintln(progress, reportPath) - - // create the path - report, err := fsx.Create(reportPath, fsx.DefaultFilePerm) - if err != nil { - return err - } - - // and write out the report + // write the machine report { - _, err := sl.Report(report) - return err + reportPath := filepath.Join(stagingDir, ReportMachinePath) + fmt.Fprintln(progress, reportPath) + + report, err := fsx.Create(reportPath, fsx.DefaultFilePerm) + if err != nil { + return err + } + + if err := sl.ReportMachine(report); err != nil { + return err + } } + + // write the plaintext report + { + reportPath := filepath.Join(stagingDir, ReportPlainPath) + fmt.Fprintln(progress, reportPath) + + report, err := fsx.Create(reportPath, fsx.DefaultFilePerm) + if err != nil { + return err + } + + if err := sl.ReportPlain(report); err != nil { + return err + } + } + + return nil }, progress, "Generating %s", Title) // if we only requested staging diff --git a/internal/dis/component/exporter/report.go b/internal/dis/component/exporter/report.go index 055502d..d77c9fd 100644 --- a/internal/dis/component/exporter/report.go +++ b/internal/dis/component/exporter/report.go @@ -13,12 +13,15 @@ func (snapshot Snapshot) String() string { builder := pools.GetBuilder() defer pools.ReleaseBuilder(builder) - snapshot.Report(builder) + snapshot.ReportPlain(builder) return builder.String() } -// Report writes a report from snapshot into w -func (snapshot Snapshot) Report(w io.Writer) (int, error) { +func (snapshot Snapshot) ReportMachine(w io.Writer) error { + return json.NewEncoder(w).Encode(snapshot) +} + +func (snapshot Snapshot) ReportPlain(w io.Writer) error { ww := &sequence.Writer{Writer: w} encoder := json.NewEncoder(ww) @@ -46,8 +49,7 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) { fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart) fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop) - fmt.Fprintf(ww, "Whitebox: %s\n", snapshot.ErrWhitebox) - fmt.Fprintf(ww, "Blackbox: %s\n", snapshot.ErrBlackbox) + fmt.Fprintf(ww, "Errors: %s\n", snapshot.Errors) io.WriteString(ww, "\n") @@ -60,7 +62,8 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) { io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n") - return ww.Sum() + _, err := ww.Sum() + return err } // Strings turns this backup into a string for the BackupReport. @@ -68,12 +71,16 @@ func (backup Backup) String() string { builder := pools.GetBuilder() defer pools.ReleaseBuilder(builder) - backup.Report(builder) + backup.ReportPlain(builder) return builder.String() } +func (backup Backup) ReportMachine(w io.Writer) error { + return json.NewEncoder(w).Encode(backup) +} + // Report formats a report for this backup, and writes it into Writer. -func (backup Backup) Report(w io.Writer) (int, error) { +func (backup Backup) ReportPlain(w io.Writer) error { cw := &sequence.Writer{Writer: w} encoder := json.NewEncoder(cw) @@ -110,5 +117,6 @@ func (backup Backup) Report(w io.Writer) (int, error) { io.WriteString(cw, "\n") - return cw.Sum() + _, err := cw.Sum() + return err } diff --git a/internal/dis/component/exporter/snapshot.go b/internal/dis/component/exporter/snapshot.go index a4cbbc0..b04e3cc 100644 --- a/internal/dis/component/exporter/snapshot.go +++ b/internal/dis/component/exporter/snapshot.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "os" "path/filepath" "time" @@ -12,8 +13,11 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/wisski" "github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/locker" "github.com/FAU-CDI/wisski-distillery/pkg/logging" + "github.com/rs/zerolog" "github.com/tkw1536/pkglib/collection" + "github.com/tkw1536/pkglib/contextx" "github.com/tkw1536/pkglib/status" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -21,6 +25,8 @@ import ( type SnapshotDescription struct { Dest string // destination path Keepalive bool // should we keep the instance alive while making the snapshot? + + Parts []string // SnapshotName()s of the components to include. } // Snapshot represents the result of generating a snapshot @@ -33,18 +39,26 @@ type Snapshot struct { EndTime time.Time // Generic Panic that may have occured - ErrPanic interface{} - ErrStart error - ErrStop error - ErrWhitebox map[string]error - ErrBlackbox map[string]error + ErrPanic interface{} + ErrStart error + ErrStop error + + // Errors holds errors for each component + Errors map[string]error + + // Logs contains logfiles for each component + Logs map[string]string // List of files included WithManifest + + // snapshotables that are running and not running + partsRunning []component.Snapshotable `json:"-"` + partsStopped []component.Snapshotable `json:"-"` } // Snapshot creates a new snapshot of this instance into dest -func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) { +func (exporter *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) { logging.LogMessage(progress, "Locking instance") if !instance.Locker().TryLock(ctx) { @@ -58,11 +72,16 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis } defer func() { logging.LogMessage(progress, "Unlocking instance") + + ctx, cancel := contextx.Anyways(ctx, time.Second) + defer cancel() + instance.Locker().Unlock(ctx) }() // setup the snapshot snapshot.Description = desc + exporter.resolveParts(ctx, desc.Parts, &snapshot) snapshot.Instance = instance.Instance // capture anything critical, and write the end time @@ -74,10 +93,15 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis logging.LogOperation(func() error { snapshot.StartTime = time.Now().UTC() - snapshot.ErrWhitebox = snapshot.makeParts(ctx, progress, snapshots, instance, false) - snapshot.ErrBlackbox = snapshot.makeParts(ctx, progress, snapshots, instance, true) + wboxerr, wboxmsg := snapshot.makeParts(ctx, progress, exporter, instance, false) + bboxerr, bboxlog := snapshot.makeParts(ctx, progress, exporter, instance, true) snapshot.EndTime = time.Now().UTC() + + // collection all the errors and logs + snapshot.Errors = collection.Append(wboxerr, bboxerr) + snapshot.Logs = collection.Append(wboxmsg, bboxlog) + return nil }, progress, "Writing snapshot files") @@ -85,7 +109,53 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis return } -func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) map[string]error { +// resolveParts resolves parts, and writes it into snapshot.Description.Parts. +// Also sets up snapshot.partsRunning and snapshot.partsStopped. +// sends a warning about unknown parts into the logger in context. +func (snapshots *Exporter) resolveParts(ctx context.Context, parts []string, snapshot *Snapshot) { + partMap := make(map[string]component.Snapshotable, len(snapshots.Dependencies.Snapshotable)) + for _, part := range snapshots.Dependencies.Snapshotable { + partMap[part.SnapshotName()] = part + } + + // filter the parts (if requested) + if len(parts) != 0 { + keys := make(map[string]struct{}, len(parts)) + for _, part := range parts { + keys[part] = struct{}{} + } + + // delete all the parts which weren't explicitly requested + for part := range partMap { + if _, ok := keys[part]; !ok { + delete(partMap, part) + } else { + delete(keys, part) + } + } + + // throw a warning for unknown parts + for key := range keys { + zerolog.Ctx(ctx).Warn().Str("part", key).Msg("ignoring unknown snapshot part") + } + } + + // sort the names of all requested parts + snapshot.Description.Parts = maps.Keys(partMap) + slices.Sort(snapshot.Description.Parts) + + // and setup the map for running and stopped parts! + for _, name := range snapshot.Description.Parts { + part := partMap[name] + if part.SnapshotNeedsRunning() { + snapshot.partsRunning = append(snapshot.partsRunning, part) + } else { + snapshot.partsStopped = append(snapshot.partsStopped, part) + } + } +} + +func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) (errmap map[string]error, logmap map[string]string) { if !needsRunning && !snapshot.Description.Keepalive { stack := instance.Barrel().Stack() @@ -106,14 +176,16 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna st.Start() defer st.Stop() - // get all the components - comps := collection.FilterClone(snapshots.Dependencies.Snapshotable, func(sc component.Snapshotable) bool { - return sc.SnapshotNeedsRunning() == needsRunning - }) + // get the components + var comps []component.Snapshotable + if needsRunning { + comps = snapshot.partsRunning + } else { + comps = snapshot.partsStopped + } - results := make(map[string]error, len(comps)) - - errors, _ := status.Group[component.Snapshotable, error]{ + // run each of the parts + errors, ids := status.Group[component.Snapshotable, error]{ PrefixString: func(item component.Snapshotable, index int) string { return fmt.Sprintf("[snapshot %q]: ", item.Name()) }, @@ -134,8 +206,29 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, sna ResultString: status.DefaultErrorString[component.Snapshotable], }.Use(st, comps) + // keep all the log files + files := st.Keep() + + // store errors and logs + errmap = make(map[string]error, len(comps)) + logmap = make(map[string]string, len(comps)) + for i, wc := range comps { - results[wc.Name()] = errors[i] + name := wc.SnapshotName() + errmap[name] = errors[i] + + // read the logfile + logfile := files[ids[i]] + bytes, err := os.ReadFile(logfile) + if err != nil { + zerolog.Ctx(ctx).Err(err).Str("component", name).Msg("unable to copy logfile") + continue + } + + // delete it, but store the content in the results + os.Remove(logfile) + logmap[name] = string(bytes) } - return results + + return }