From 630da9e12fa1bab5951d9660246db3500a27cefc Mon Sep 17 00:00:00 2001
From: Tom Wiesing
Date: Mon, 3 Oct 2022 11:22:45 +0200
Subject: [PATCH] {backup,snapshot}: Log and display in control
---
cmd/backup.go | 33 +++++++--
cmd/snapshot.go | 35 +++++++--
internal/component/control/html/index.html | 26 +++++++
internal/component/control/html/instance.html | 27 +++++++
.../component/control/html/static/instance.js | 6 +-
internal/component/control/info.go | 46 ++++++++----
internal/component/instances/wisski_log.go | 73 +++++++++++++++++++
internal/component/instances/wisski_status.go | 8 ++
internal/component/snapshots/backup.go | 1 -
internal/component/snapshots/log.go | 17 +++++
internal/component/snapshots/manager.go | 6 +-
.../snapshots/prune.go} | 25 ++++---
internal/component/sql/connect.go | 1 +
internal/component/sql/update.go | 5 ++
internal/dis/component.go | 1 +
internal/models/snapshot.go | 18 +++++
pkg/slicesx/slicesx.go | 10 +++
17 files changed, 294 insertions(+), 44 deletions(-)
create mode 100644 internal/component/instances/wisski_log.go
create mode 100644 internal/component/snapshots/log.go
rename internal/{dis/backup_prune.go => component/snapshots/prune.go} (50%)
create mode 100644 internal/models/snapshot.go
diff --git a/cmd/backup.go b/cmd/backup.go
index 361c403..0088ca7 100644
--- a/cmd/backup.go
+++ b/cmd/backup.go
@@ -4,6 +4,7 @@ import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/core"
+ "github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@@ -44,7 +45,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
if !bk.NoPrune {
defer logging.LogOperation(func() error {
- return dis.PruneBackups(context.IOStream)
+ return dis.SnapshotManager().PruneBackups(context.IOStream)
}, context.IOStream, "Pruning old backups")
}
@@ -81,28 +82,44 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
}
context.Println(sPath)
+ var logEntry models.Snapshot
logging.LogOperation(func() error {
backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{
Dest: sPath,
- Auto: bk.Positionals.Dest == "",
ConcurrentSnapshots: bk.ConcurrentSnapshots,
})
backup.WriteReport(dis.Core.Environment, context.IOStream)
+ logEntry = backup.LogEntry()
return nil
}, context.IOStream, "Generating Backup")
- // if we requested to only have a staging area, then we are done
- if bk.StagingOnly {
- context.Printf("Wrote %s\n", sPath)
- return nil
- }
-
// create the archive path
archivePath := bk.Positionals.Dest
if archivePath == "" {
archivePath = dis.SnapshotManager().NewArchivePath("")
}
+ // do the logging
+ if bk.Positionals.Dest == "" {
+ defer logging.LogOperation(func() error {
+ if bk.StagingOnly {
+ logEntry.Path = sPath
+ logEntry.Packed = false
+ } else {
+ logEntry.Path = archivePath
+ logEntry.Packed = true
+ }
+
+ return dis.Instances().AddSnapshotLog(logEntry)
+ }, context.IOStream, "Writing Log Entry")
+ }
+
+ // if we requested to only have a staging area, then we are done
+ if bk.StagingOnly {
+ context.Printf("Wrote %s\n", sPath)
+ return nil
+ }
+
// and write everything into it!
var count int64
if err := logging.LogOperation(func() error {
diff --git a/cmd/snapshot.go b/cmd/snapshot.go
index 0ff0b8b..ca061f5 100644
--- a/cmd/snapshot.go
+++ b/cmd/snapshot.go
@@ -6,6 +6,7 @@ import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/core"
+ "github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@@ -42,6 +43,8 @@ var errSnapshotFailed = exit.Error{
}
func (bi snapshot) Run(context wisski_distillery.Context) error {
+ // TODO: Cleanup this code!
+
dis := context.Environment
instance, err := dis.Instances().WissKI(bi.Positionals.Slug)
if err != nil {
@@ -52,6 +55,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// determine the target path for the archive
var sPath string
+
if !bi.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
@@ -86,6 +90,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// TODO: Allow skipping backups of individual parts and make them concurrent!
// take a snapshot into the staging area!
+ var logEntry models.Snapshot
logging.LogOperation(func() error {
sreport := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{
Dest: sPath,
@@ -95,19 +100,35 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// write out the report, ignoring any errors!
sreport.WriteReport(dis.Core.Environment, context.IOStream)
+ logEntry = sreport.LogEntry()
+
return nil
}, context.IOStream, "Generating Snapshot")
- // if we requested to only have a staging area, then we are done
- if bi.StagingOnly {
- context.Printf("Wrote %s\n", sPath)
- return nil
- }
-
// create the archive path
archivePath := bi.Positionals.Dest
if archivePath == "" {
- archivePath = dis.SnapshotManager().NewArchivePath(instance.Slug)
+ archivePath = dis.SnapshotManager().NewArchivePath("")
+ }
+
+ // do the logging
+ if bi.Positionals.Dest == "" {
+ defer logging.LogOperation(func() error {
+ if bi.StagingOnly {
+ logEntry.Path = sPath
+ logEntry.Packed = false
+ } else {
+ logEntry.Path = archivePath
+ logEntry.Packed = true
+ }
+
+ return dis.Instances().AddSnapshotLog(logEntry)
+ }, context.IOStream, "Writing Log Entry")
+ }
+ // if we requested to only have a staging area, then we are done
+ if bi.StagingOnly {
+ context.Printf("Wrote %s\n", sPath)
+ return nil
}
// and write everything into it!
diff --git a/internal/component/control/html/index.html b/internal/component/control/html/index.html
index 1b81a68..d6a82e7 100644
--- a/internal/component/control/html/index.html
+++ b/internal/component/control/html/index.html
@@ -26,6 +26,32 @@
GraphDB Database Prefix: {{.Config.GraphDBRepoPrefix}}
Bookkeeping Database: {{.Config.DistilleryDatabase}}
+
+ Backups:
+
+
+
+ | Path |
+ Created |
+ Packed |
+
+
+
+ {{ range .Info.Backups }}
+
+
+ {{ .Path }}
+ |
+
+ {{ .Created.Format "2006-01-02T15:04:05Z07:00" }}
+ |
+
+ {{ .Packed }}
+ |
+
+ {{ end}}
+
+
Instances
diff --git a/internal/component/control/html/instance.html b/internal/component/control/html/instance.html
index 00ff40d..440bec9 100644
--- a/internal/component/control/html/instance.html
+++ b/internal/component/control/html/instance.html
@@ -36,6 +36,33 @@
GraphDBRepository: {{ .Instance.GraphDBRepository }}
GraphDBUsername: {{ .Instance.GraphDBUsername }}
+
+ Snapshots:
+
+
+
+ | Path |
+ Created |
+ Packed |
+
+
+
+ {{ range .Info.Snapshots }}
+
+
+ {{ .Path }}
+ |
+
+ {{ .Created.Format "2006-01-02T15:04:05Z07:00" }}
+ |
+
+ {{ .Packed }}
+ |
+
+ {{ end }}
+
+
+