rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

{backup,snapshot}: Rename common code to export

Browse source
This commit is contained in:
Tom Wiesing 2022-10-07 22:12:14 +02:00
parent b3a827e042
commit bf57c0d5a6
No known key found for this signature in database
14 changed files with 145 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

22
cmd/backup.go
View file

@ -9,9 +9,9 @@ import (
)
// Backup is the 'backup' command
var Backup wisski_distillery.Command = backupC{}
var Backup wisski_distillery.Command = backup{}
type backupC struct {
type backup struct {
NoPrune bool `short:"n" long:"no-prune" description:"Do not prune older backup archives"`
StagingOnly bool `short:"s" long:"staging-only" description:"Do not package into a backup archive, but only create a staging directory"`
ConcurrentSnapshots int `short:"c" long:"concurrent-snapshots" description:"Maximum number of concurrent snapshots" default:"2"`
@ -20,7 +20,7 @@ type backupC struct {
} `positional-args:"true"`
}
func (backupC) Description() wisski_distillery.Description {
func (backup) Description() wisski_distillery.Description {
return wisski_distillery.Description{
Requirements: core.Requirements{
NeedsDistillery: true,
@ -35,29 +35,25 @@ var errBackupFailed = exit.Error{
ExitCode: exit.ExitGeneric,
}
func (bk backupC) Run(context wisski_distillery.Context) error {
func (bk backup) Run(context wisski_distillery.Context) error {
dis := context.Environment
// prune old backups
if !bk.NoPrune {
defer logging.LogOperation(func() error {
return dis.SnapshotManager().PruneBackups(context.IOStream)
return dis.SnapshotManager().PruneExports(context.IOStream)
}, context.IOStream, "Pruning old backups")
}
// do the handling
err := dis.SnapshotManager().HandleSnapshotLike(context.IOStream, snapshots.SnapshotFlags{
err := dis.SnapshotManager().MakeExport(context.IOStream, snapshots.ExportTask{
Dest: bk.Positionals.Dest,
Slug: "",
Title: "Backup",
StagingOnly: bk.StagingOnly,
Do: func(dest string) snapshots.SnapshotLike {
backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{
Dest: dest,
Instance: nil,
BackupDescription: snapshots.BackupDescription{
ConcurrentSnapshots: bk.ConcurrentSnapshots,
})
return &backup
},
})

7
cmd/config.go
View file

@ -8,10 +8,9 @@ import (
// Config is the configuration command
var Config wisski_distillery.Command = cfg{}
type cfg struct {
}
type cfg struct{}
func (s cfg) Description() wisski_distillery.Description {
func (c cfg) Description() wisski_distillery.Description {
return wisski_distillery.Description{
Requirements: core.Requirements{
NeedsDistillery: true,
@ -21,7 +20,7 @@ func (s cfg) Description() wisski_distillery.Description {
}
}
func (s cfg) Run(context wisski_distillery.Context) error {
func (cfg) Run(context wisski_distillery.Context) error {
context.Printf("%#v", context.Environment.Config)
return nil
}

19
cmd/snapshot.go
View file

@ -35,28 +35,21 @@ var errSnapshotFailed = exit.Error{
ExitCode: exit.ExitGeneric,
}
func (bi snapshot) Run(context wisski_distillery.Context) error {
func (sn snapshot) Run(context wisski_distillery.Context) error {
dis := context.Environment
// find the instance!
instance, err := dis.Instances().WissKI(bi.Positionals.Slug)
instance, err := dis.Instances().WissKI(sn.Positionals.Slug)
if err != nil {
return err
}
// do a snapshot of it!
err = dis.SnapshotManager().HandleSnapshotLike(context.IOStream, snapshots.SnapshotFlags{
Dest: bi.Positionals.Dest,
Slug: bi.Positionals.Slug,
Title: "Snapshot",
StagingOnly: bi.StagingOnly,
err = dis.SnapshotManager().MakeExport(context.IOStream, snapshots.ExportTask{
Dest: sn.Positionals.Dest,
StagingOnly: sn.StagingOnly,
Do: func(dest string) snapshots.SnapshotLike {
snapshot := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{
Dest: dest,
})
return &snapshot
},
Instance: &instance,
})
if err != nil {

4
internal/component/info/index.go
View file

@ -28,7 +28,7 @@ type indexPageContext struct {
RunningCount int
StoppedCount int
Backups []models.Snapshot
Backups []models.Export
}
func (info *Info) indexPageAPI(r *http.Request) (idx indexPageContext, err error) {
@ -61,7 +61,7 @@ func (info *Info) indexPageAPI(r *http.Request) (idx indexPageContext, err error
// get the log entries
group.Go(func() (err error) {
idx.Backups, err = info.Instances.SnapshotLogFor("")
idx.Backups, err = info.Instances.ExportLogFor("")
return
})

18
internal/component/info/socket.go
View file

@ -44,23 +44,13 @@ func (info *Info) serverSocketSnapshot(slug string, writer *status.LineBuffer) {
}
{
err := info.SnapshotManager.HandleSnapshotLike(
err := info.SnapshotManager.MakeExport(
stream,
snapshots.SnapshotFlags{
snapshots.ExportTask{
Dest: "",
Slug: slug,
Title: "Snapshot",
Instance: &wissKI,
StagingOnly: false,
Do: func(dest string) snapshots.SnapshotLike {
snapshot := info.SnapshotManager.NewSnapshot(
wissKI,
stream,
snapshots.SnapshotDescription{
Dest: dest,
},
)
return &snapshot
},
},
)
if err != nil {

39
internal/component/instances/wisski_log.go
View file

@ -6,36 +6,37 @@ import (
"github.com/tkw1536/goprogram/lib/collection"
)
// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug.
// An empty slug returns the log of backups.
func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snapshot, err error) {
snapshots, err = instances.SnapshotLog()
// ExportLogFor retrieves (and prunes) the ExportLog.
// Slug determines if entries for Backups (empty slug)
// or a specific Instance (non-empty slug) are returned.
func (instances *Instances) ExportLogFor(slug string) (exports []models.Export, err error) {
exports, err = instances.ExportLog()
if err != nil {
return nil, err
}
return collection.Filter(snapshots, func(s models.Snapshot) bool {
return collection.Filter(exports, func(s models.Export) bool {
return s.Slug == slug
}), nil
}
// SnapshotLog retrieves (and prunes) all entries in the snapshot log.
func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
// ExportLog retrieves (and prunes) all entries in the snapshot log.
func (instances *Instances) ExportLog() ([]models.Export, error) {
// query the table!
table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
table, err := instances.SQL.QueryTable(false, models.ExportTable)
if err != nil {
return nil, err
}
// find all the snapshots
var snapshots []models.Snapshot
res := table.Find(&snapshots)
// find all the exports
var exports []models.Export
res := table.Find(&exports)
if res.Error != nil {
return nil, res.Error
}
// partition out the snapshots that have been deleted!
parts := collection.Partition(snapshots, func(s models.Snapshot) bool {
// partition out the exports that have been deleted!
parts := collection.Partition(exports, func(s models.Export) bool {
_, err := instances.Core.Environment.Stat(s.Path)
return !environment.IsNotExist(err)
})
@ -52,20 +53,20 @@ func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
}
// Snapshots returns the list of snapshots of this WissKI
func (wisski *WissKI) Snapshots() (snapshots []models.Snapshot, err error) {
return wisski.instances.SnapshotLogFor(wisski.Slug)
func (wisski *WissKI) Snapshots() (snapshots []models.Export, err error) {
return wisski.instances.ExportLogFor(wisski.Slug)
}
// AddSnapshotLog adds a log entry for the provided entry
func (instances *Instances) AddSnapshotLog(snapshot models.Snapshot) error {
// AddToExportLog adds the provided export to the log.
func (instances *Instances) AddToExportLog(export models.Export) error {
// find the table
table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
table, err := instances.SQL.QueryTable(false, models.ExportTable)
if err != nil {
return err
}
// and save it!
res := table.Create(&snapshot)
res := table.Create(&export)
if res.Error != nil {
return res.Error
}

6
internal/component/instances/wisski_status.go
View file

@ -21,10 +21,10 @@ type WissKIInfo struct {
LastRebuild time.Time
// List of backups made
Snapshots []models.Snapshot
Snapshots []models.Export
// WissKI content information
NoPrefixes bool
NoPrefixes bool // TODO: Move this into the database
Prefixes []string // list of prefixes
Pathbuilders map[string]string // all the pathbuilders
}
@ -39,7 +39,7 @@ func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
info.Slug = wisski.Slug
info.URL = wisski.URL().String()
// dynamic properties, TODO: Add more properties here!
// dynamic properties
var group errgroup.Group
// quick check if this wisski is running

2
internal/component/pool_meta.go
View file

@ -8,7 +8,7 @@ import (
"github.com/tkw1536/goprogram/lib/reflectx"
)
var metaCache sync.Map
var metaCache sync.Map // Map[reflect.Type]meta
// getMeta gets the component belonging to a component type
func getMeta[C Component]() meta {

98
internal/component/snapshots/iface.go
View file

@ -4,6 +4,7 @@ import (
"io"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
@ -12,43 +13,66 @@ import (
"github.com/tkw1536/goprogram/stream"
)
type SnapshotFlags struct {
// ExportTask describes a task that makes either a [Backup] or a [Snapshot].
// See [Manager.MakeExport]
type ExportTask struct {
// Dest is the destination path to write the backup to.
// When empty, this is created automatically in the staging or archive directory.
Dest string
Slug string
Title string // "Backup" or "Snapshot"
// By default, a .tar.gz file is generated.
// To generated an unpacked directory, set [StagingOnly] to true.
StagingOnly bool
Do func(dest string) SnapshotLike
// Instance is the instance to generate a snapshot of.
// To generate a backup, leave this to be nil.
Instance *instances.WissKI
// BackupDescriptions and SnapshotDescriptions further specitfy options for the export.
// The Dest parameter is ignored, and updated automatically.
BackupDescription BackupDescription
SnapshotDescription SnapshotDescription
}
type SnapshotLike interface {
LogEntry() models.Snapshot
// export is implemented by [Backup] and [Snapshot]
type export interface {
LogEntry() models.Export
Report(w io.Writer) (int, error)
}
func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags SnapshotFlags) (err error) {
// MakeExport performs an export task as described by flags.
// Output is directed to the provided io.
func (manager *Manager) MakeExport(io stream.IOStream, task ExportTask) (err error) {
// extract parameters
Title := "Backup"
Slug := ""
if task.Instance != nil {
Title = "Snapshot"
Slug = task.Instance.Slug
}
// determine target paths
logging.LogMessage(context, "Determining target paths")
logging.LogMessage(io, "Determining target paths")
var stagingDir, archivePath string
if flags.StagingOnly {
stagingDir = flags.Dest
if task.StagingOnly {
stagingDir = task.Dest
} else {
archivePath = flags.Dest
archivePath = task.Dest
}
if stagingDir == "" {
stagingDir, err = manager.NewStagingDir(flags.Slug)
stagingDir, err = manager.NewStagingDir(Slug)
if err != nil {
return err
}
}
if !flags.StagingOnly && archivePath == "" {
archivePath = manager.NewArchivePath(flags.Slug)
if !task.StagingOnly && archivePath == "" {
archivePath = manager.NewArchivePath(Slug)
}
context.Printf("Staging Directory: %s\n", stagingDir)
context.Printf("Archive Path: %s\n", archivePath)
io.Printf("Staging Directory: %s\n", stagingDir)
io.Printf("Archive Path: %s\n", archivePath)
// create the staging directory
logging.LogMessage(context, "Creating staging directory")
logging.LogMessage(io, "Creating staging directory")
err = manager.Environment.Mkdir(stagingDir, environment.DefaultDirPerm)
if !environment.IsExist(err) && err != nil {
return err
@ -56,9 +80,9 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh
// if it was requested to not do staging only
// we need the staging directory to be deleted at the end
if !flags.StagingOnly {
if !task.StagingOnly {
defer func() {
logging.LogMessage(context, "Removing staging directory")
logging.LogMessage(io, "Removing staging directory")
manager.Environment.RemoveAll(stagingDir)
}()
}
@ -66,17 +90,25 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh
// create the actual snapshot or backup
// write out the report
// and retain a log entry
var entry models.Snapshot
var entry models.Export
logging.LogOperation(func() error {
// do the snapshot!
sl := flags.Do(stagingDir)
var sl export
if task.Instance == nil {
task.BackupDescription.Dest = stagingDir
backup := manager.NewBackup(io, task.BackupDescription)
sl = &backup
} else {
task.SnapshotDescription.Dest = stagingDir
snapshot := manager.NewSnapshot(*task.Instance, io, task.SnapshotDescription)
sl = &snapshot
}
// create a log entry
entry = sl.LogEntry()
// find the report path
reportPath := filepath.Join(stagingDir, "report.txt")
context.Println(reportPath)
io.Println(reportPath)
// create the path
report, err := manager.Environment.Create(reportPath, environment.DefaultFilePerm)
@ -89,28 +121,28 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh
_, err := sl.Report(report)
return err
}
}, context, "Generating %s", flags.Title)
}, io, "Generating %s", Title)
// if we only requested staging
// all that is left is to write the log entry
if flags.StagingOnly {
logging.LogMessage(context, "Writing Log Entry")
if task.StagingOnly {
logging.LogMessage(io, "Writing Log Entry")
// write out the log entry
entry.Path = stagingDir
entry.Packed = false
manager.Instances.AddSnapshotLog(entry)
manager.Instances.AddToExportLog(entry)
context.Printf("Wrote %s\n", stagingDir)
io.Printf("Wrote %s\n", stagingDir)
return nil
}
// package everything up as an archive!
if err := logging.LogOperation(func() error {
var count int64
defer func() { context.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }()
defer func() { io.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }()
st := status.NewWithCompat(context.Stdout, 1)
st := status.NewWithCompat(io.Stdout, 1)
st.Start()
defer st.Stop()
@ -119,15 +151,15 @@ func (manager *Manager) HandleSnapshotLike(context stream.IOStream, flags Snapsh
})
return err
}, context, "Writing archive"); err != nil {
}, io, "Writing archive"); err != nil {
return err
}
// write out the log entry
logging.LogMessage(context, "Writing Log Entry")
logging.LogMessage(io, "Writing Log Entry")
entry.Path = archivePath
entry.Packed = true
manager.Instances.AddSnapshotLog(entry)
manager.Instances.AddToExportLog(entry)
// and we're done!
return nil

8
internal/component/snapshots/log.go
View file

@ -2,15 +2,15 @@ package snapshots
import "github.com/FAU-CDI/wisski-distillery/internal/models"
func (backup *Backup) LogEntry() models.Snapshot {
return models.Snapshot{
func (backup *Backup) LogEntry() models.Export {
return models.Export{
Created: backup.StartTime,
Slug: "",
}
}
func (snapshot *Snapshot) LogEntry() models.Snapshot {
return models.Snapshot{
func (snapshot *Snapshot) LogEntry() models.Export {
return models.Export{
Created: snapshot.StartTime,
Slug: snapshot.Instance.Slug,
}

9
internal/component/snapshots/prune.go
View file

@ -7,13 +7,14 @@ import (
"github.com/tkw1536/goprogram/stream"
)
// ShouldPrune determines if a file with the provided modtime
// ShouldPrune determines if a file with the provided modification time should be
// removed from the export log.
func (manager *Manager) ShouldPrune(modtime time.Time) bool {
return time.Since(modtime) > time.Duration(manager.Config.MaxBackupAge)*24*time.Hour
}
// Prune prunes all backups and snapshots older than the maximum backup age
func (manager *Manager) PruneBackups(io stream.IOStream) error {
// Prune prunes all old exports
func (manager *Manager) PruneExports(io stream.IOStream) error {
sPath := manager.ArchivePath()
// list all the files
@ -49,6 +50,6 @@ func (manager *Manager) PruneBackups(io stream.IOStream) error {
}
// prune the snapshot log!
_, err = manager.Instances.SnapshotLog()
_, err = manager.Instances.ExportLog()
return err
}

4
internal/component/sql/update.go
View file

@ -93,8 +93,8 @@ func (sql *SQL) Update(io stream.IOStream) error {
},
{
"snapshot",
&models.Snapshot{},
models.SnapshotTable,
&models.Export{},
models.ExportTable,
},
}

18
internal/models/export.go Normal file
View file

@ -0,0 +1,18 @@
package models
import "time"
// ExportTable is the name of the table the [Export] model is stored in.
// NOTE(twiesing): It is called snapshot for legacy reasons
const ExportTable = "snapshot"
// Export represents an entry in the export log
type Export struct {
Pk uint `gorm:"column:pk;primaryKey"`
Slug string `gorm:"column:slug"` // slug of instance
Created time.Time `gorm:"column:created"` // time the backup was created
Path string `gorm:"column:path;not null"` // path the export is stored at
Packed bool `gorm:"column:packed;not null"` // was the export packed, or was it staging only?
}

18
internal/models/snapshot.go
View file

@ -1,18 +0,0 @@
package models
import "time"
// SnapshotTable is the name of the table the [SnapshotLog] model is stored in
const SnapshotTable = "snapshot"
// Snapshot represents an entry in the snapshot log
type Snapshot struct {
Pk uint `gorm:"column:pk;primaryKey"`
Slug string `gorm:"column:slug"` // slug of instance
Created time.Time `gorm:"column:created"` // time the backup was created
Path string `gorm:"column:path;not null"` // path the backup is stored at
Packed bool `gorm:"column:packed;not null"` // was the backup packed, or was it staging only?
}