{backup,snapshot}: Log and display in control

cmd/backup.go

@@ -4,6 +4,7 @@ import (
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
 	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@@ -44,7 +45,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
 
 	if !bk.NoPrune {
 		defer logging.LogOperation(func() error {
-			return dis.PruneBackups(context.IOStream)
+			return dis.SnapshotManager().PruneBackups(context.IOStream)
 		}, context.IOStream, "Pruning old backups")
 	}
 
@@ -81,28 +82,44 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
 	}
 	context.Println(sPath)
 
+	var logEntry models.Snapshot
 	logging.LogOperation(func() error {
 		backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{
 			Dest:                sPath,
-			Auto:                bk.Positionals.Dest == "",
 			ConcurrentSnapshots: bk.ConcurrentSnapshots,
 		})
 		backup.WriteReport(dis.Core.Environment, context.IOStream)
+		logEntry = backup.LogEntry()
 		return nil
 	}, context.IOStream, "Generating Backup")
 
-	// if we requested to only have a staging area, then we are done
-	if bk.StagingOnly {
-		context.Printf("Wrote %s\n", sPath)
-		return nil
-	}
-
 	// create the archive path
 	archivePath := bk.Positionals.Dest
 	if archivePath == "" {
 		archivePath = dis.SnapshotManager().NewArchivePath("")
 	}
 
+	// do the logging
+	if bk.Positionals.Dest == "" {
+		defer logging.LogOperation(func() error {
+			if bk.StagingOnly {
+				logEntry.Path = sPath
+				logEntry.Packed = false
+			} else {
+				logEntry.Path = archivePath
+				logEntry.Packed = true
+			}
+
+			return dis.Instances().AddSnapshotLog(logEntry)
+		}, context.IOStream, "Writing Log Entry")
+	}
+
+	// if we requested to only have a staging area, then we are done
+	if bk.StagingOnly {
+		context.Printf("Wrote %s\n", sPath)
+		return nil
+	}
+
 	// and write everything into it!
 	var count int64
 	if err := logging.LogOperation(func() error {

cmd/snapshot.go

@@ -6,6 +6,7 @@ import (
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
 	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@@ -42,6 +43,8 @@ var errSnapshotFailed = exit.Error{
 }
 
 func (bi snapshot) Run(context wisski_distillery.Context) error {
+	// TODO: Cleanup this code!
+
 	dis := context.Environment
 	instance, err := dis.Instances().WissKI(bi.Positionals.Slug)
 	if err != nil {
@@ -52,6 +55,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
 
 	// determine the target path for the archive
 	var sPath string
+
 	if !bi.StagingOnly {
 		// regular mode: create a temporary staging directory
 		logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
@@ -86,6 +90,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
 	// TODO: Allow skipping backups of individual parts and make them concurrent!
 
 	// take a snapshot into the staging area!
+	var logEntry models.Snapshot
 	logging.LogOperation(func() error {
 		sreport := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{
 			Dest:      sPath,
@@ -95,19 +100,35 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
 		// write out the report, ignoring any errors!
 		sreport.WriteReport(dis.Core.Environment, context.IOStream)
 
+		logEntry = sreport.LogEntry()
+
 		return nil
 	}, context.IOStream, "Generating Snapshot")
 
-	// if we requested to only have a staging area, then we are done
-	if bi.StagingOnly {
-		context.Printf("Wrote %s\n", sPath)
-		return nil
-	}
-
 	// create the archive path
 	archivePath := bi.Positionals.Dest
 	if archivePath == "" {
-		archivePath = dis.SnapshotManager().NewArchivePath(instance.Slug)
+		archivePath = dis.SnapshotManager().NewArchivePath("")
+	}
+
+	// do the logging
+	if bi.Positionals.Dest == "" {
+		defer logging.LogOperation(func() error {
+			if bi.StagingOnly {
+				logEntry.Path = sPath
+				logEntry.Packed = false
+			} else {
+				logEntry.Path = archivePath
+				logEntry.Packed = true
+			}
+
+			return dis.Instances().AddSnapshotLog(logEntry)
+		}, context.IOStream, "Writing Log Entry")
+	}
+	// if we requested to only have a staging area, then we are done
+	if bi.StagingOnly {
+		context.Printf("Wrote %s\n", sPath)
+		return nil
 	}
 
 	// and write everything into it!

internal/component/control/html/index.html

@@ -26,6 +26,32 @@
     <b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
     <hr />
     <b>Bookkeeping Database:</b> <code>{{.Config.DistilleryDatabase}}</code><br />
+    <hr />
+    <b>Backups:</b>
+    <table>
+        <thead>
+            <tr>
+                <th>Path</th>
+                <th>Created</th>
+                <th>Packed</th>
+            </tr>
+        </thead>
+        <tbody>
+            {{ range .Info.Backups }}
+            <tr>
+                <td>
+                    <code class="path">{{ .Path }}</code>
+                </td>
+                <td>
+                    <code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
+                </td>
+                <td>
+                    {{ .Packed }}
+                </td>
+            </tr>
+        {{ end}}
+        </tbody>
+    </table>
 </p>
 
 <h2 id="instances">Instances</h2>

internal/component/control/html/instance.html

@@ -36,6 +36,33 @@
     <hr />
 	<b>GraphDBRepository:</b> <code>{{ .Instance.GraphDBRepository }}</code> <br />
 	<b>GraphDBUsername:</b> <code>{{ .Instance.GraphDBUsername }}</code> <br />
+    <hr />
+    <b>Snapshots:</b>
+    <table>
+        <thead>
+            <tr>
+                <th>Path</th>
+                <th>Created</th>
+                <th>Packed</th>
+            </tr>
+        </thead>
+        <tbody>
+            {{ range .Info.Snapshots }}
+            <tr>
+                <td>
+                    <code class="path">{{ .Path }}</code>
+                </td>
+                <td>
+                    <code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
+                </td>
+                <td>
+                    {{ .Packed }}
+                </td>
+            </tr>
+        {{ end }}
+        </tbody>
+    </table>
+    <hr />
 </p>
 
 <footer>

internal/component/control/html/static/instance.js

@@ -1,6 +1,10 @@
 const types = {
     "date": (element) => {
-        return (new Date(element.innerText)).toString()
+        return (new Date(element.innerText)).toISOString()
+    },
+    "path": (element) => {
+        const text = element.innerText.split("/");
+        return text[text.length - 1];
     },
     "pathbuilders": (element) => {
         const pathbuilders = window.pathbuilders; // read from context!

internal/component/control/info.go

@@ -62,28 +62,43 @@ type disIndex struct {
 
 	Config *config.Config
 
-	Instances    []instances.WissKIInfo
+	Instances []instances.WissKIInfo
+
 	TotalCount   int
 	RunningCount int
 	StoppedCount int
+
+	Backups []models.Snapshot
 }
 
 func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
-	// load instances
+	var group errgroup.Group
-	idx.Instances, err = dis.allinstances(r)
-	if err != nil {
-		return
-	}
 
-	// count how many are running and how many are stopped
+	group.Go(func() error {
-	for _, i := range idx.Instances {
+		// load instances
-		if i.Running {
+		idx.Instances, err = dis.allinstances(r)
-			idx.RunningCount++
+		if err != nil {
-		} else {
+			return err
-			idx.StoppedCount++
 		}
-	}
+
-	idx.TotalCount = len(idx.Instances)
+		// count how many are running and how many are stopped
+		for _, i := range idx.Instances {
+			if i.Running {
+				idx.RunningCount++
+			} else {
+				idx.StoppedCount++
+			}
+		}
+		idx.TotalCount = len(idx.Instances)
+
+		return nil
+	})
+
+	// get the log entries
+	group.Go(func() (err error) {
+		idx.Backups, err = dis.Instances.SnapshotLogFor("")
+		return
+	})
 
 	// get the static properties
 	idx.Config = dis.Config
@@ -91,6 +106,9 @@ func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
 	// current time
 	idx.Time = time.Now().UTC()
 
+	// wait for everything!
+	group.Wait()
+
 	return
 }
 

internal/component/instances/wisski_log.go

@@ -0,0 +1,73 @@
+package instances
+
+import (
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
+)
+
+// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug.
+// An empty slug returns the log of backups.
+func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snapshot, err error) {
+	snapshots, err = instances.SnapshotLog()
+	if err != nil {
+		return nil, err
+	}
+
+	return slicesx.Filter(snapshots, func(s models.Snapshot) bool {
+		return s.Slug == slug
+	}), nil
+}
+
+// SnapshotLog retrieves (and prunes) all entries in the snapshot log.
+func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
+	// query the table!
+	table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
+	if err != nil {
+		return nil, err
+	}
+
+	// find all the snapshots
+	var snapshots []models.Snapshot
+	res := table.Find(&snapshots)
+	if res.Error != nil {
+		return nil, res.Error
+	}
+
+	// partition out the snapshots that have been deleted!
+	parts := slicesx.Partition(snapshots, func(s models.Snapshot) bool {
+		_, err := instances.Core.Environment.Stat(s.Path)
+		return !environment.IsNotExist(err)
+	})
+
+	// go and delete them!
+	if len(parts[false]) > 0 {
+		if err := table.Delete(parts[false]).Error; err != nil {
+			return nil, err
+		}
+	}
+
+	// return the ones that still exist
+	return parts[true], nil
+}
+
+// Snapshots returns the list of snapshots of this WissKI
+func (wisski *WissKI) Snapshots() (snapshots []models.Snapshot, err error) {
+	return wisski.instances.SnapshotLogFor(wisski.Slug)
+}
+
+// AddSnapshotLog adds a log entry for the provided entry
+func (instances *Instances) AddSnapshotLog(snapshot models.Snapshot) error {
+	// find the table
+	table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
+	if err != nil {
+		return err
+	}
+
+	// and save it!
+	res := table.Create(&snapshot)
+	if res.Error != nil {
+		return res.Error
+	}
+	return nil
+}

internal/component/instances/wisski_status.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/sync/errgroup"
 )
@@ -20,6 +21,9 @@ type WissKIInfo struct {
 	Running     bool
 	LastRebuild time.Time
 
+	// List of backups made
+	Snapshots []models.Snapshot
+
 	// WissKI content information
 	Prefixes     []string          // list of prefixes
 	Pathbuilders map[string]string // all the pathbuilders
@@ -59,6 +63,10 @@ func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
 			info.Prefixes, _ = wisski.Prefixes()
 			return nil
 		})
+		group.Go(func() error {
+			info.Snapshots, _ = wisski.Snapshots()
+			return nil
+		})
 	}
 
 	err = group.Wait()

internal/component/snapshots/backup.go

@@ -44,7 +44,6 @@ type Backup struct {
 // BackupDescription provides a description for a backup
 type BackupDescription struct {
 	Dest string // Destination path
-	Auto bool   // Was the path created automatically?
 
 	ConcurrentSnapshots int // maximum number of concurrent snapshots
 }

internal/component/snapshots/log.go

@@ -0,0 +1,17 @@
+package snapshots
+
+import "github.com/FAU-CDI/wisski-distillery/internal/models"
+
+func (backup *Backup) LogEntry() models.Snapshot {
+	return models.Snapshot{
+		Created: backup.StartTime,
+		Slug:    "",
+	}
+}
+
+func (snapshot *Snapshot) LogEntry() models.Snapshot {
+	return models.Snapshot{
+		Created: snapshot.StartTime,
+		Slug:    snapshot.Instance.Slug,
+	}
+}

internal/component/snapshots/manager.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/password"
@@ -15,6 +16,8 @@ import (
 // Manager manages snapshots and backups
 type Manager struct {
 	component.ComponentBase
+
+	SQL       *sql.SQL
 	Instances *instances.Instances
 
 	Snapshotable []component.Snapshotable
@@ -54,7 +57,7 @@ func (dis *Manager) NewArchivePath(prefix string) (path string) {
 // newSnapshot name returns a new basename for a snapshot with the provided prefix.
 // The name is guaranteed to be unique within this process.
 func (*Manager) newSnapshotName(prefix string) string {
-	suffix, _ := password.Password(64) // silently ignore any errors!
+	suffix, _ := password.Password(10) // silently ignore any errors!
 	if prefix == "" {
 		prefix = "backup"
 	} else {
@@ -68,7 +71,6 @@ func (*Manager) newSnapshotName(prefix string) string {
 func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
 	for path == "" || environment.IsExist(err) {
 		path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
-		fmt.Println("path =>", prefix, "err => ", err)
 		err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
 	}
 	if err != nil {

internal/component/snapshots/prune.go

@@ -1,4 +1,4 @@
-package dis
+package snapshots
 
 import (
 	"path/filepath"
@@ -8,16 +8,16 @@ import (
 )
 
 // ShouldPrune determines if a file with the provided modtime
-func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
+func (manager *Manager) ShouldPrune(modtime time.Time) bool {
-	return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
+	return time.Since(modtime) > time.Duration(manager.Config.MaxBackupAge)*24*time.Hour
 }
 
-// PruneBackups prunes all backups older than the maximum backup age
+// Prune prunes all backups and snapshots older than the maximum backup age
-func (dis *Distillery) PruneBackups(io stream.IOStream) error {
+func (manager *Manager) PruneBackups(io stream.IOStream) error {
-	sPath := dis.SnapshotManager().ArchivePath()
+	sPath := manager.ArchivePath()
 
 	// list all the files
-	entries, err := dis.Core.Environment.ReadDir(sPath)
+	entries, err := manager.Core.Environment.ReadDir(sPath)
 	if err != nil {
 		return err
 	}
@@ -35,17 +35,20 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
 		}
 
 		// check if it should be pruned!
-		if !dis.ShouldPrune(info.ModTime()) {
+		if !manager.ShouldPrune(info.ModTime()) {
 			continue
 		}
 
 		// assemble path, and then remove the file!
 		path := filepath.Join(sPath, entry.Name())
-		io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
+		io.Printf("Removing %s cause it is older than %d days", path, manager.Config.MaxBackupAge)
 
-		if err := dis.Core.Environment.Remove(path); err != nil {
+		if err := manager.Core.Environment.Remove(path); err != nil {
 			return err
 		}
 	}
-	return nil
+
+	// prune the snapshot log!
+	_, err = manager.Instances.SnapshotLog()
+	return err
 }

internal/component/sql/connect.go

@@ -90,6 +90,7 @@ func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
 
 // WaitQueryTable waits for a connection to succeed via QueryTable
 func (sql *SQL) WaitQueryTable() error {
+	// TODO: Establish a convention on when to wait for this!
 	n := stream.FromDebug()
 	return wait.Wait(func() bool {
 		_, err := sql.QueryTable(true, models.InstanceTable)

internal/component/sql/update.go

@@ -91,6 +91,11 @@ func (sql *SQL) Update(io stream.IOStream) error {
 			&models.Metadatum{},
 			models.MetadataTable,
 		},
+		{
+			"snapshot",
+			&models.Snapshot{},
+			models.SnapshotTable,
+		},
 	}
 
 	// migrate all of the tables!

internal/dis/component.go

@@ -73,6 +73,7 @@ func (dis *Distillery) cInstances(thread int32) *instances.Instances {
 
 func (dis *Distillery) cSnapshotManager(thread int32) *snapshots.Manager {
 	return component.PutComponent(&dis.pool, thread, dis.Core, func(snapshots *snapshots.Manager, thread int32) {
+		snapshots.SQL = dis.cSQL(thread)
 		snapshots.Instances = dis.cInstances(thread)
 		snapshots.Snapshotable = dis.cSnapshotable(thread)
 		snapshots.Backupable = dis.cBackupable(thread)

internal/models/snapshot.go

@@ -0,0 +1,18 @@
+package models
+
+import "time"
+
+// SnapshotTable is the name of the table the [SnapshotLog] model is stored in
+const SnapshotTable = "snapshot"
+
+// Snapshot represents an entry in the snapshot log
+type Snapshot struct {
+	Pk uint `gorm:"column:pk;primaryKey"`
+
+	Slug    string    `gorm:"column:slug"`    // slug of instance
+	Created time.Time `gorm:"column:created"` // time the backup was created
+
+	Path   string `gorm:"column:path;not null"`   // path the backup is stored at
+	Packed bool   `gorm:"column:packed;not null"` // was the backup packed, or was it staging only?
+
+}

pkg/slicesx/slicesx.go

@@ -26,6 +26,16 @@ func Filter[T any](values []T, filter func(T) bool) []T {
 	return results
 }
 
+// Partition partitions values in T by the given functions.
+func Partition[T any, P comparable](values []T, partition func(value T) P) map[P][]T {
+	result := make(map[P][]T)
+	for _, v := range values {
+		part := partition(v)
+		result[part] = append(result[part], v)
+	}
+	return result
+}
+
 // FilterClone is like [Filter], but creates a new slice
 func FilterClone[T any](values []T, filter func(T) bool) (results []T) {
 	for _, value := range values {