{backup,snapshot}: Log and display in control

internal/component/control/html/index.html

@@ -26,6 +26,32 @@
     <b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
     <hr />
     <b>Bookkeeping Database:</b> <code>{{.Config.DistilleryDatabase}}</code><br />
+    <hr />
+    <b>Backups:</b>
+    <table>
+        <thead>
+            <tr>
+                <th>Path</th>
+                <th>Created</th>
+                <th>Packed</th>
+            </tr>
+        </thead>
+        <tbody>
+            {{ range .Info.Backups }}
+            <tr>
+                <td>
+                    <code class="path">{{ .Path }}</code>
+                </td>
+                <td>
+                    <code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
+                </td>
+                <td>
+                    {{ .Packed }}
+                </td>
+            </tr>
+        {{ end}}
+        </tbody>
+    </table>
 </p>
 
 <h2 id="instances">Instances</h2>

internal/component/control/html/instance.html

@@ -36,6 +36,33 @@
     <hr />
 	<b>GraphDBRepository:</b> <code>{{ .Instance.GraphDBRepository }}</code> <br />
 	<b>GraphDBUsername:</b> <code>{{ .Instance.GraphDBUsername }}</code> <br />
+    <hr />
+    <b>Snapshots:</b>
+    <table>
+        <thead>
+            <tr>
+                <th>Path</th>
+                <th>Created</th>
+                <th>Packed</th>
+            </tr>
+        </thead>
+        <tbody>
+            {{ range .Info.Snapshots }}
+            <tr>
+                <td>
+                    <code class="path">{{ .Path }}</code>
+                </td>
+                <td>
+                    <code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
+                </td>
+                <td>
+                    {{ .Packed }}
+                </td>
+            </tr>
+        {{ end }}
+        </tbody>
+    </table>
+    <hr />
 </p>
 
 <footer>

internal/component/control/html/static/instance.js

@@ -1,6 +1,10 @@
 const types = {
     "date": (element) => {
-        return (new Date(element.innerText)).toString()
+        return (new Date(element.innerText)).toISOString()
+    },
+    "path": (element) => {
+        const text = element.innerText.split("/");
+        return text[text.length - 1];
     },
     "pathbuilders": (element) => {
         const pathbuilders = window.pathbuilders; // read from context!

internal/component/control/info.go

@@ -62,28 +62,43 @@ type disIndex struct {
 
 	Config *config.Config
 
-	Instances    []instances.WissKIInfo
+	Instances []instances.WissKIInfo
+
 	TotalCount   int
 	RunningCount int
 	StoppedCount int
+
+	Backups []models.Snapshot
 }
 
 func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
-	// load instances
-	idx.Instances, err = dis.allinstances(r)
-	if err != nil {
-		return
-	}
+	var group errgroup.Group
 
-	// count how many are running and how many are stopped
-	for _, i := range idx.Instances {
-		if i.Running {
-			idx.RunningCount++
-		} else {
-			idx.StoppedCount++
+	group.Go(func() error {
+		// load instances
+		idx.Instances, err = dis.allinstances(r)
+		if err != nil {
+			return err
 		}
-	}
-	idx.TotalCount = len(idx.Instances)
+
+		// count how many are running and how many are stopped
+		for _, i := range idx.Instances {
+			if i.Running {
+				idx.RunningCount++
+			} else {
+				idx.StoppedCount++
+			}
+		}
+		idx.TotalCount = len(idx.Instances)
+
+		return nil
+	})
+
+	// get the log entries
+	group.Go(func() (err error) {
+		idx.Backups, err = dis.Instances.SnapshotLogFor("")
+		return
+	})
 
 	// get the static properties
 	idx.Config = dis.Config
@@ -91,6 +106,9 @@ func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
 	// current time
 	idx.Time = time.Now().UTC()
 
+	// wait for everything!
+	group.Wait()
+
 	return
 }
 

internal/component/instances/wisski_log.go

@@ -0,0 +1,73 @@
+package instances
+
+import (
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
+)
+
+// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug.
+// An empty slug returns the log of backups.
+func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snapshot, err error) {
+	snapshots, err = instances.SnapshotLog()
+	if err != nil {
+		return nil, err
+	}
+
+	return slicesx.Filter(snapshots, func(s models.Snapshot) bool {
+		return s.Slug == slug
+	}), nil
+}
+
+// SnapshotLog retrieves (and prunes) all entries in the snapshot log.
+func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
+	// query the table!
+	table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
+	if err != nil {
+		return nil, err
+	}
+
+	// find all the snapshots
+	var snapshots []models.Snapshot
+	res := table.Find(&snapshots)
+	if res.Error != nil {
+		return nil, res.Error
+	}
+
+	// partition out the snapshots that have been deleted!
+	parts := slicesx.Partition(snapshots, func(s models.Snapshot) bool {
+		_, err := instances.Core.Environment.Stat(s.Path)
+		return !environment.IsNotExist(err)
+	})
+
+	// go and delete them!
+	if len(parts[false]) > 0 {
+		if err := table.Delete(parts[false]).Error; err != nil {
+			return nil, err
+		}
+	}
+
+	// return the ones that still exist
+	return parts[true], nil
+}
+
+// Snapshots returns the list of snapshots of this WissKI
+func (wisski *WissKI) Snapshots() (snapshots []models.Snapshot, err error) {
+	return wisski.instances.SnapshotLogFor(wisski.Slug)
+}
+
+// AddSnapshotLog adds a log entry for the provided entry
+func (instances *Instances) AddSnapshotLog(snapshot models.Snapshot) error {
+	// find the table
+	table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
+	if err != nil {
+		return err
+	}
+
+	// and save it!
+	res := table.Create(&snapshot)
+	if res.Error != nil {
+		return res.Error
+	}
+	return nil
+}

internal/component/instances/wisski_status.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
 	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/sync/errgroup"
 )
@@ -20,6 +21,9 @@ type WissKIInfo struct {
 	Running     bool
 	LastRebuild time.Time
 
+	// List of backups made
+	Snapshots []models.Snapshot
+
 	// WissKI content information
 	Prefixes     []string          // list of prefixes
 	Pathbuilders map[string]string // all the pathbuilders
@@ -59,6 +63,10 @@ func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
 			info.Prefixes, _ = wisski.Prefixes()
 			return nil
 		})
+		group.Go(func() error {
+			info.Snapshots, _ = wisski.Snapshots()
+			return nil
+		})
 	}
 
 	err = group.Wait()

internal/component/snapshots/backup.go

@@ -44,7 +44,6 @@ type Backup struct {
 // BackupDescription provides a description for a backup
 type BackupDescription struct {
 	Dest string // Destination path
-	Auto bool   // Was the path created automatically?
 
 	ConcurrentSnapshots int // maximum number of concurrent snapshots
 }

internal/component/snapshots/log.go

@@ -0,0 +1,17 @@
+package snapshots
+
+import "github.com/FAU-CDI/wisski-distillery/internal/models"
+
+func (backup *Backup) LogEntry() models.Snapshot {
+	return models.Snapshot{
+		Created: backup.StartTime,
+		Slug:    "",
+	}
+}
+
+func (snapshot *Snapshot) LogEntry() models.Snapshot {
+	return models.Snapshot{
+		Created: snapshot.StartTime,
+		Slug:    snapshot.Instance.Slug,
+	}
+}

internal/component/snapshots/manager.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
 	"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/password"
@@ -15,6 +16,8 @@ import (
 // Manager manages snapshots and backups
 type Manager struct {
 	component.ComponentBase
+
+	SQL       *sql.SQL
 	Instances *instances.Instances
 
 	Snapshotable []component.Snapshotable
@@ -54,7 +57,7 @@ func (dis *Manager) NewArchivePath(prefix string) (path string) {
 // newSnapshot name returns a new basename for a snapshot with the provided prefix.
 // The name is guaranteed to be unique within this process.
 func (*Manager) newSnapshotName(prefix string) string {
-	suffix, _ := password.Password(64) // silently ignore any errors!
+	suffix, _ := password.Password(10) // silently ignore any errors!
 	if prefix == "" {
 		prefix = "backup"
 	} else {
@@ -68,7 +71,6 @@ func (*Manager) newSnapshotName(prefix string) string {
 func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
 	for path == "" || environment.IsExist(err) {
 		path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
-		fmt.Println("path =>", prefix, "err => ", err)
 		err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
 	}
 	if err != nil {

internal/component/snapshots/prune.go

@@ -0,0 +1,54 @@
+package snapshots
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// ShouldPrune determines if a file with the provided modtime
+func (manager *Manager) ShouldPrune(modtime time.Time) bool {
+	return time.Since(modtime) > time.Duration(manager.Config.MaxBackupAge)*24*time.Hour
+}
+
+// Prune prunes all backups and snapshots older than the maximum backup age
+func (manager *Manager) PruneBackups(io stream.IOStream) error {
+	sPath := manager.ArchivePath()
+
+	// list all the files
+	entries, err := manager.Core.Environment.ReadDir(sPath)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		// skip directories
+		if entry.IsDir() {
+			continue
+		}
+
+		// grab info about the file
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		// check if it should be pruned!
+		if !manager.ShouldPrune(info.ModTime()) {
+			continue
+		}
+
+		// assemble path, and then remove the file!
+		path := filepath.Join(sPath, entry.Name())
+		io.Printf("Removing %s cause it is older than %d days", path, manager.Config.MaxBackupAge)
+
+		if err := manager.Core.Environment.Remove(path); err != nil {
+			return err
+		}
+	}
+
+	// prune the snapshot log!
+	_, err = manager.Instances.SnapshotLog()
+	return err
+}

internal/component/sql/connect.go

@@ -90,6 +90,7 @@ func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
 
 // WaitQueryTable waits for a connection to succeed via QueryTable
 func (sql *SQL) WaitQueryTable() error {
+	// TODO: Establish a convention on when to wait for this!
 	n := stream.FromDebug()
 	return wait.Wait(func() bool {
 		_, err := sql.QueryTable(true, models.InstanceTable)

internal/component/sql/update.go

@@ -91,6 +91,11 @@ func (sql *SQL) Update(io stream.IOStream) error {
 			&models.Metadatum{},
 			models.MetadataTable,
 		},
+		{
+			"snapshot",
+			&models.Snapshot{},
+			models.SnapshotTable,
+		},
 	}
 
 	// migrate all of the tables!