From bf57c0d5a6db693eb1a1b2d5d9a4181b5dab186e Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Fri, 7 Oct 2022 22:12:14 +0200 Subject: [PATCH] {backup,snapshot}: Rename common code to export --- cmd/backup.go | 24 ++--- cmd/config.go | 7 +- cmd/snapshot.go | 19 ++-- internal/component/info/index.go | 4 +- internal/component/info/socket.go | 20 +--- internal/component/instances/wisski_log.go | 39 +++---- internal/component/instances/wisski_status.go | 6 +- internal/component/pool_meta.go | 2 +- internal/component/snapshots/iface.go | 100 ++++++++++++------ internal/component/snapshots/log.go | 8 +- internal/component/snapshots/prune.go | 9 +- internal/component/sql/update.go | 4 +- internal/models/export.go | 18 ++++ internal/models/snapshot.go | 18 ---- 14 files changed, 145 insertions(+), 133 deletions(-) create mode 100644 internal/models/export.go delete mode 100644 internal/models/snapshot.go diff --git a/cmd/backup.go b/cmd/backup.go index 59026c2..fd8a5fa 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -9,9 +9,9 @@ import ( ) // Backup is the 'backup' command -var Backup wisski_distillery.Command = backupC{} +var Backup wisski_distillery.Command = backup{} -type backupC struct { +type backup struct { NoPrune bool `short:"n" long:"no-prune" description:"Do not prune older backup archives"` StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"` ConcurrentSnapshots int `short:"c" long:"concurrent-snapshots" description:"Maximum number of concurrent snapshots" default:"2"` @@ -20,7 +20,7 @@ type backupC struct { } `positional-args:"true"` } -func (backupC) Description() wisski_distillery.Description { +func (backup) Description() wisski_distillery.Description { return wisski_distillery.Description{ Requirements: core.Requirements{ NeedsDistillery: true, @@ -35,29 +35,25 @@ var errBackupFailed = exit.Error{ ExitCode: exit.ExitGeneric, } -func (bk backupC) Run(context wisski_distillery.Context) error { +func (bk backup) Run(context wisski_distillery.Context) error { dis := context.Environment // prune old backups if !bk.NoPrune { defer logging.LogOperation(func() error { - return dis.SnapshotManager().PruneBackups(context.IOStream) + return dis.SnapshotManager().PruneExports(context.IOStream) }, context.IOStream, "Pruning old backups") } // do the handling - err := dis.SnapshotManager().HandleSnapshotLike(context.IOStream, snapshots.SnapshotFlags{ + err := dis.SnapshotManager().MakeExport(context.IOStream, snapshots.ExportTask{ Dest: bk.Positionals.Dest, - Slug: "", - Title: "Backup", StagingOnly: bk.StagingOnly, - Do: func(dest string) snapshots.SnapshotLike { - backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{ - Dest: dest, - ConcurrentSnapshots: bk.ConcurrentSnapshots, - }) - return &backup + Instance: nil, + + BackupDescription: snapshots.BackupDescription{ + ConcurrentSnapshots: bk.ConcurrentSnapshots, }, }) diff --git a/cmd/config.go b/cmd/config.go index 28cdf2d..871dc09 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -8,10 +8,9 @@ import ( // Config is the configuration command var Config wisski_distillery.Command = cfg{} -type cfg struct { -} +type cfg struct{} -func (s cfg) Description() wisski_distillery.Description { +func (c cfg) Description() wisski_distillery.Description { return wisski_distillery.Description{ Requirements: core.Requirements{ NeedsDistillery: true, @@ -21,7 +20,7 @@ func (s cfg) Description() wisski_distillery.Description { } } -func (s cfg) Run(context wisski_distillery.Context) error { +func (cfg) Run(context wisski_distillery.Context) error { context.Printf("%#v", context.Environment.Config) return nil } diff --git a/cmd/snapshot.go b/cmd/snapshot.go index bde0a5a..b389a99 100644 --- a/cmd/snapshot.go +++ b/cmd/snapshot.go @@ -35,28 +35,21 @@ var errSnapshotFailed = exit.Error{ ExitCode: exit.ExitGeneric, } -func (bi snapshot) Run(context wisski_distillery.Context) error { +func (sn snapshot) Run(context wisski_distillery.Context) error { dis := context.Environment // find the instance! - instance, err := dis.Instances().WissKI(bi.Positionals.Slug) + instance, err := dis.Instances().WissKI(sn.Positionals.Slug) if err != nil { return err } // do a snapshot of it! - err = dis.SnapshotManager().HandleSnapshotLike(context.IOStream, snapshots.SnapshotFlags{ - Dest: bi.Positionals.Dest, - Slug: bi.Positionals.Slug, - Title: "Snapshot", - StagingOnly: bi.StagingOnly, + err = dis.SnapshotManager().MakeExport(context.IOStream, snapshots.ExportTask{ + Dest: sn.Positionals.Dest, + StagingOnly: sn.StagingOnly, - Do: func(dest string) snapshots.SnapshotLike { - snapshot := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{ - Dest: dest, - }) - return &snapshot - }, + Instance: &instance, }) if err != nil { diff --git a/internal/component/info/index.go b/internal/component/info/index.go index 14f94ab..056688e 100644 --- a/internal/component/info/index.go +++ b/internal/component/info/index.go @@ -28,7 +28,7 @@ type indexPageContext struct { RunningCount int StoppedCount int - Backups []models.Snapshot + Backups []models.Export } func (info *Info) indexPageAPI(r *http.Request) (idx indexPageContext, err error) { @@ -61,7 +61,7 @@ func (info *Info) indexPageAPI(r *http.Request) (idx indexPageContext, err error // get the log entries group.Go(func() (err error) { - idx.Backups, err = info.Instances.SnapshotLogFor("") + idx.Backups, err = info.Instances.ExportLogFor("") return }) diff --git a/internal/component/info/socket.go b/internal/component/info/socket.go index a23ebce..33ac48f 100644 --- a/internal/component/info/socket.go +++ b/internal/component/info/socket.go @@ -44,23 +44,13 @@ func (info *Info) serverSocketSnapshot(slug string, writer *status.LineBuffer) { } { - err := info.SnapshotManager.HandleSnapshotLike( + err := info.SnapshotManager.MakeExport( stream, - snapshots.SnapshotFlags{ - Dest: "", - Slug: slug, - Title: "Snapshot", + snapshots.ExportTask{ + Dest: "", + Instance: &wissKI, + StagingOnly: false, - Do: func(dest string) snapshots.SnapshotLike { - snapshot := info.SnapshotManager.NewSnapshot( - wissKI, - stream, - snapshots.SnapshotDescription{ - Dest: dest, - }, - ) - return &snapshot - }, }, ) if err != nil { diff --git a/internal/component/instances/wisski_log.go b/internal/component/instances/wisski_log.go index 62b5567..cd3b5ae 100644 --- a/internal/component/instances/wisski_log.go +++ b/internal/component/instances/wisski_log.go @@ -6,36 +6,37 @@ import ( "github.com/tkw1536/goprogram/lib/collection" ) -// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug. -// An empty slug returns the log of backups. -func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snapshot, err error) { - snapshots, err = instances.SnapshotLog() +// ExportLogFor retrieves (and prunes) the ExportLog. +// Slug determines if entries for Backups (empty slug) +// or a specific Instance (non-empty slug) are returned. +func (instances *Instances) ExportLogFor(slug string) (exports []models.Export, err error) { + exports, err = instances.ExportLog() if err != nil { return nil, err } - return collection.Filter(snapshots, func(s models.Snapshot) bool { + return collection.Filter(exports, func(s models.Export) bool { return s.Slug == slug }), nil } -// SnapshotLog retrieves (and prunes) all entries in the snapshot log. -func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) { +// ExportLog retrieves (and prunes) all entries in the snapshot log. +func (instances *Instances) ExportLog() ([]models.Export, error) { // query the table! - table, err := instances.SQL.QueryTable(false, models.SnapshotTable) + table, err := instances.SQL.QueryTable(false, models.ExportTable) if err != nil { return nil, err } - // find all the snapshots - var snapshots []models.Snapshot - res := table.Find(&snapshots) + // find all the exports + var exports []models.Export + res := table.Find(&exports) if res.Error != nil { return nil, res.Error } - // partition out the snapshots that have been deleted! - parts := collection.Partition(snapshots, func(s models.Snapshot) bool { + // partition out the exports that have been deleted! + parts := collection.Partition(exports, func(s models.Export) bool { _, err := instances.Core.Environment.Stat(s.Path) return !environment.IsNotExist(err) }) @@ -52,20 +53,20 @@ func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) { } // Snapshots returns the list of snapshots of this WissKI -func (wisski *WissKI) Snapshots() (snapshots []models.Snapshot, err error) { - return wisski.instances.SnapshotLogFor(wisski.Slug) +func (wisski *WissKI) Snapshots() (snapshots []models.Export, err error) { + return wisski.instances.ExportLogFor(wisski.Slug) } -// AddSnapshotLog adds a log entry for the provided entry -func (instances *Instances) AddSnapshotLog(snapshot models.Snapshot) error { +// AddToExportLog adds the provided export to the log. +func (instances *Instances) AddToExportLog(export models.Export) error { // find the table - table, err := instances.SQL.QueryTable(false, models.SnapshotTable) + table, err := instances.SQL.QueryTable(false, models.ExportTable) if err != nil { return err } // and save it! - res := table.Create(&snapshot) + res := table.Create(&export) if res.Error != nil { return res.Error } diff --git a/internal/component/instances/wisski_status.go b/internal/component/instances/wisski_status.go index 6096c11..ce51912 100644 --- a/internal/component/instances/wisski_status.go +++ b/internal/component/instances/wisski_status.go @@ -21,10 +21,10 @@ type WissKIInfo struct { LastRebuild time.Time // List of backups made - Snapshots []models.Snapshot + Snapshots []models.Export // WissKI content information - NoPrefixes bool + NoPrefixes bool // TODO: Move this into the database Prefixes []string // list of prefixes Pathbuilders map[string]string // all the pathbuilders } @@ -39,7 +39,7 @@ func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) { info.Slug = wisski.Slug info.URL = wisski.URL().String() - // dynamic properties, TODO: Add more properties here! + // dynamic properties var group errgroup.Group // quick check if this wisski is running diff --git a/internal/component/pool_meta.go b/internal/component/pool_meta.go index df4339f..6b66193 100644 --- a/internal/component/pool_meta.go +++ b/internal/component/pool_meta.go @@ -8,7 +8,7 @@ import ( "github.com/tkw1536/goprogram/lib/reflectx" ) -var metaCache sync.Map +var metaCache sync.Map // Map[reflect.Type]meta // getMeta gets the component belonging to a component type func getMeta[C Component]() meta { diff --git a/internal/component/snapshots/iface.go b/internal/component/snapshots/iface.go index cb5f265..8accf3e 100644 --- a/internal/component/snapshots/iface.go +++ b/internal/component/snapshots/iface.go @@ -4,6 +4,7 @@ import ( "io" "path/filepath" + "github.com/FAU-CDI/wisski-distillery/internal/component/instances" "github.com/FAU-CDI/wisski-distillery/internal/models" "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/logging" @@ -12,43 +13,66 @@ import ( "github.com/tkw1536/goprogram/stream" ) -type SnapshotFlags struct { - Dest string - Slug string - Title string // "Backup" or "Snapshot" +// ExportTask describes a task that makes either a [Backup] or a [Snapshot]. +// See [Manager.MakeExport] +type ExportTask struct { + // Dest is the destination path to write the backup to. + // When empty, this is created automatically in the staging or archive directory. + Dest string + + // By default, a .tar.gz file is generated. + // To generated an unpacked directory, set [StagingOnly] to true. StagingOnly bool - Do func(dest string) SnapshotLike + // Instance is the instance to generate a snapshot of. + // To generate a backup, leave this to be nil. + Instance *instances.WissKI + + // BackupDescriptions and SnapshotDescriptions further specitfy options for the export. + // The Dest parameter is ignored, and updated automatically. + BackupDescription BackupDescription + SnapshotDescription SnapshotDescription } -type SnapshotLike interface { - LogEntry() models.Snapshot +// export is implemented by [Backup] and [Snapshot] +type export interface { + LogEntry() models.Export Report(w io.Writer) (int, error) } -func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags SnapshotFlags) (err error) { +// MakeExport performs an export task as described by flags. +// Output is directed to the provided io. +func (manager *Manager) MakeExport(io stream.IOStream, task ExportTask) (err error) { + // extract parameters + Title := "Backup" + Slug := "" + if task.Instance != nil { + Title = "Snapshot" + Slug = task.Instance.Slug + } + // determine target paths - logging.LogMessage(context, "Determining target paths") + logging.LogMessage(io, "Determining target paths") var stagingDir, archivePath string - if flags.StagingOnly { - stagingDir = flags.Dest + if task.StagingOnly { + stagingDir = task.Dest } else { - archivePath = flags.Dest + archivePath = task.Dest } if stagingDir == "" { - stagingDir, err = manager.NewStagingDir(flags.Slug) + stagingDir, err = manager.NewStagingDir(Slug) if err != nil { return err } } - if !flags.StagingOnly && archivePath == "" { - archivePath = manager.NewArchivePath(flags.Slug) + if !task.StagingOnly && archivePath == "" { + archivePath = manager.NewArchivePath(Slug) } - context.Printf("Staging Directory: %s\n", stagingDir) - context.Printf("Archive Path: %s\n", archivePath) + io.Printf("Staging Directory: %s\n", stagingDir) + io.Printf("Archive Path: %s\n", archivePath) // create the staging directory - logging.LogMessage(context, "Creating staging directory") + logging.LogMessage(io, "Creating staging directory") err = manager.Environment.Mkdir(stagingDir, environment.DefaultDirPerm) if !environment.IsExist(err) && err != nil { return err @@ -56,9 +80,9 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh // if it was requested to not do staging only // we need the staging directory to be deleted at the end - if !flags.StagingOnly { + if !task.StagingOnly { defer func() { - logging.LogMessage(context, "Removing staging directory") + logging.LogMessage(io, "Removing staging directory") manager.Environment.RemoveAll(stagingDir) }() } @@ -66,17 +90,25 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh // create the actual snapshot or backup // write out the report // and retain a log entry - var entry models.Snapshot + var entry models.Export logging.LogOperation(func() error { - // do the snapshot! - sl := flags.Do(stagingDir) + var sl export + if task.Instance == nil { + task.BackupDescription.Dest = stagingDir + backup := manager.NewBackup(io, task.BackupDescription) + sl = &backup + } else { + task.SnapshotDescription.Dest = stagingDir + snapshot := manager.NewSnapshot(*task.Instance, io, task.SnapshotDescription) + sl = &snapshot + } // create a log entry entry = sl.LogEntry() // find the report path reportPath := filepath.Join(stagingDir, "report.txt") - context.Println(reportPath) + io.Println(reportPath) // create the path report, err := manager.Environment.Create(reportPath, environment.DefaultFilePerm) @@ -89,28 +121,28 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh _, err := sl.Report(report) return err } - }, context, "Generating %s", flags.Title) + }, io, "Generating %s", Title) // if we only requested staging // all that is left is to write the log entry - if flags.StagingOnly { - logging.LogMessage(context, "Writing Log Entry") + if task.StagingOnly { + logging.LogMessage(io, "Writing Log Entry") // write out the log entry entry.Path = stagingDir entry.Packed = false - manager.Instances.AddSnapshotLog(entry) + manager.Instances.AddToExportLog(entry) - context.Printf("Wrote %s\n", stagingDir) + io.Printf("Wrote %s\n", stagingDir) return nil } // package everything up as an archive! if err := logging.LogOperation(func() error { var count int64 - defer func() { context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }() + defer func() { io.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }() - st := status.NewWithCompat(context.Stdout, 1) + st := status.NewWithCompat(io.Stdout, 1) st.Start() defer st.Stop() @@ -119,15 +151,15 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh }) return err - }, context, "Writing archive"); err != nil { + }, io, "Writing archive"); err != nil { return err } // write out the log entry - logging.LogMessage(context, "Writing Log Entry") + logging.LogMessage(io, "Writing Log Entry") entry.Path = archivePath entry.Packed = true - manager.Instances.AddSnapshotLog(entry) + manager.Instances.AddToExportLog(entry) // and we're done! return nil diff --git a/internal/component/snapshots/log.go b/internal/component/snapshots/log.go index 2b9d3ff..aac8e94 100644 --- a/internal/component/snapshots/log.go +++ b/internal/component/snapshots/log.go @@ -2,15 +2,15 @@ package snapshots import "github.com/FAU-CDI/wisski-distillery/internal/models" -func (backup *Backup) LogEntry() models.Snapshot { - return models.Snapshot{ +func (backup *Backup) LogEntry() models.Export { + return models.Export{ Created: backup.StartTime, Slug: "", } } -func (snapshot *Snapshot) LogEntry() models.Snapshot { - return models.Snapshot{ +func (snapshot *Snapshot) LogEntry() models.Export { + return models.Export{ Created: snapshot.StartTime, Slug: snapshot.Instance.Slug, } diff --git a/internal/component/snapshots/prune.go b/internal/component/snapshots/prune.go index 48600f8..c99b53c 100644 --- a/internal/component/snapshots/prune.go +++ b/internal/component/snapshots/prune.go @@ -7,13 +7,14 @@ import ( "github.com/tkw1536/goprogram/stream" ) -// ShouldPrune determines if a file with the provided modtime +// ShouldPrune determines if a file with the provided modification time should be +// removed from the export log. func (manager *Manager) ShouldPrune(modtime time.Time) bool { return time.Since(modtime) > time.Duration(manager.Config.MaxBackupAge)*24*time.Hour } -// Prune prunes all backups and snapshots older than the maximum backup age -func (manager *Manager) PruneBackups(io stream.IOStream) error { +// Prune prunes all old exports +func (manager *Manager) PruneExports(io stream.IOStream) error { sPath := manager.ArchivePath() // list all the files @@ -49,6 +50,6 @@ func (manager *Manager) PruneBackups(io stream.IOStream) error { } // prune the snapshot log! - _, err = manager.Instances.SnapshotLog() + _, err = manager.Instances.ExportLog() return err } diff --git a/internal/component/sql/update.go b/internal/component/sql/update.go index 3f3b15f..4200095 100644 --- a/internal/component/sql/update.go +++ b/internal/component/sql/update.go @@ -93,8 +93,8 @@ func (sql *SQL) Update(io stream.IOStream) error { }, { "snapshot", - &models.Snapshot{}, - models.SnapshotTable, + &models.Export{}, + models.ExportTable, }, } diff --git a/internal/models/export.go b/internal/models/export.go new file mode 100644 index 0000000..f5ffb27 --- /dev/null +++ b/internal/models/export.go @@ -0,0 +1,18 @@ +package models + +import "time" + +// ExportTable is the name of the table the [Export] model is stored in. +// NOTE(twiesing): It is called snapshot for legacy reasons +const ExportTable = "snapshot" + +// Export represents an entry in the export log +type Export struct { + Pk uint `gorm:"column:pk;primaryKey"` + + Slug string `gorm:"column:slug"` // slug of instance + Created time.Time `gorm:"column:created"` // time the backup was created + + Path string `gorm:"column:path;not null"` // path the export is stored at + Packed bool `gorm:"column:packed;not null"` // was the export packed, or was it staging only? +} diff --git a/internal/models/snapshot.go b/internal/models/snapshot.go deleted file mode 100644 index ead57ba..0000000 --- a/internal/models/snapshot.go +++ /dev/null @@ -1,18 +0,0 @@ -package models - -import "time" - -// SnapshotTable is the name of the table the [SnapshotLog] model is stored in -const SnapshotTable = "snapshot" - -// Snapshot represents an entry in the snapshot log -type Snapshot struct { - Pk uint `gorm:"column:pk;primaryKey"` - - Slug string `gorm:"column:slug"` // slug of instance - Created time.Time `gorm:"column:created"` // time the backup was created - - Path string `gorm:"column:path;not null"` // path the backup is stored at - Packed bool `gorm:"column:packed;not null"` // was the backup packed, or was it staging only? - -}