rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

{backup,snapshot}: Log and display in control

Browse source
This commit is contained in:
Tom Wiesing 2022-10-03 11:22:45 +02:00
parent 3b112f1b8e
commit 630da9e12f
No known key found for this signature in database
17 changed files with 294 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

33
cmd/backup.go
View file

@ -4,6 +4,7 @@ import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@ -44,7 +45,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
if !bk.NoPrune {
defer logging.LogOperation(func() error {
return dis.PruneBackups(context.IOStream)
return dis.SnapshotManager().PruneBackups(context.IOStream)
}, context.IOStream, "Pruning old backups")
}
@ -81,28 +82,44 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
}
context.Println(sPath)
var logEntry models.Snapshot
logging.LogOperation(func() error {
backup := dis.SnapshotManager().NewBackup(context.IOStream, snapshots.BackupDescription{
Dest: sPath,
Auto: bk.Positionals.Dest == "",
ConcurrentSnapshots: bk.ConcurrentSnapshots,
})
backup.WriteReport(dis.Core.Environment, context.IOStream)
logEntry = backup.LogEntry()
return nil
}, context.IOStream, "Generating Backup")
// if we requested to only have a staging area, then we are done
if bk.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
// create the archive path
archivePath := bk.Positionals.Dest
if archivePath == "" {
archivePath = dis.SnapshotManager().NewArchivePath("")
}
// do the logging
if bk.Positionals.Dest == "" {
defer logging.LogOperation(func() error {
if bk.StagingOnly {
logEntry.Path = sPath
logEntry.Packed = false
} else {
logEntry.Path = archivePath
logEntry.Packed = true
}
return dis.Instances().AddSnapshotLog(logEntry)
}, context.IOStream, "Writing Log Entry")
}
// if we requested to only have a staging area, then we are done
if bk.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
// and write everything into it!
var count int64
if err := logging.LogOperation(func() error {

35
cmd/snapshot.go
View file

@ -6,6 +6,7 @@ import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@ -42,6 +43,8 @@ var errSnapshotFailed = exit.Error{
}
func (bi snapshot) Run(context wisski_distillery.Context) error {
// TODO: Cleanup this code!
dis := context.Environment
instance, err := dis.Instances().WissKI(bi.Positionals.Slug)
if err != nil {
@ -52,6 +55,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// determine the target path for the archive
var sPath string
if !bi.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
@ -86,6 +90,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// TODO: Allow skipping backups of individual parts and make them concurrent!
// take a snapshot into the staging area!
var logEntry models.Snapshot
logging.LogOperation(func() error {
sreport := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{
Dest: sPath,
@ -95,19 +100,35 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// write out the report, ignoring any errors!
sreport.WriteReport(dis.Core.Environment, context.IOStream)
logEntry = sreport.LogEntry()
return nil
}, context.IOStream, "Generating Snapshot")
// if we requested to only have a staging area, then we are done
if bi.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
// create the archive path
archivePath := bi.Positionals.Dest
if archivePath == "" {
archivePath = dis.SnapshotManager().NewArchivePath(instance.Slug)
archivePath = dis.SnapshotManager().NewArchivePath("")
}
// do the logging
if bi.Positionals.Dest == "" {
defer logging.LogOperation(func() error {
if bi.StagingOnly {
logEntry.Path = sPath
logEntry.Packed = false
} else {
logEntry.Path = archivePath
logEntry.Packed = true
}
return dis.Instances().AddSnapshotLog(logEntry)
}, context.IOStream, "Writing Log Entry")
}
// if we requested to only have a staging area, then we are done
if bi.StagingOnly {
context.Printf("Wrote %s\n", sPath)
return nil
}
// and write everything into it!

26
internal/component/control/html/index.html
View file

@ -26,6 +26,32 @@
<b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
<hr />
<b>Bookkeeping Database:</b> <code>{{.Config.DistilleryDatabase}}</code><br />
<hr />
<b>Backups:</b>
<table>
<thead>
<tr>
<th>Path</th>
<th>Created</th>
<th>Packed</th>
</tr>
</thead>
<tbody>
{{ range .Info.Backups }}
<tr>
<td>
<code class="path">{{ .Path }}</code>
</td>
<td>
<code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
</td>
<td>
{{ .Packed }}
</td>
</tr>
{{ end}}
</tbody>
</table>
</p>
<h2 id="instances">Instances</h2>

27
internal/component/control/html/instance.html
View file

@ -36,6 +36,33 @@
<hr />
<b>GraphDBRepository:</b> <code>{{ .Instance.GraphDBRepository }}</code> <br />
<b>GraphDBUsername:</b> <code>{{ .Instance.GraphDBUsername }}</code> <br />
<hr />
<b>Snapshots:</b>
<table>
<thead>
<tr>
<th>Path</th>
<th>Created</th>
<th>Packed</th>
</tr>
</thead>
<tbody>
{{ range .Info.Snapshots }}
<tr>
<td>
<code class="path">{{ .Path }}</code>
</td>
<td>
<code class="date">{{ .Created.Format "2006-01-02T15:04:05Z07:00" }}</code>
</td>
<td>
{{ .Packed }}
</td>
</tr>
{{ end }}
</tbody>
</table>
<hr />
</p>
<footer>

6
internal/component/control/html/static/instance.js
View file

@ -1,6 +1,10 @@
const types = {
"date": (element) => {
return (new Date(element.innerText)).toString()
return (new Date(element.innerText)).toISOString()
},
"path": (element) => {
const text = element.innerText.split("/");
return text[text.length - 1];
},
"pathbuilders": (element) => {
const pathbuilders = window.pathbuilders; // read from context!

46
internal/component/control/info.go
View file

@ -62,28 +62,43 @@ type disIndex struct {
Config *config.Config
Instances []instances.WissKIInfo
Instances []instances.WissKIInfo
TotalCount int
RunningCount int
StoppedCount int
Backups []models.Snapshot
}
func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
// load instances
idx.Instances, err = dis.allinstances(r)
if err != nil {
return
}
var group errgroup.Group
// count how many are running and how many are stopped
for _, i := range idx.Instances {
if i.Running {
idx.RunningCount++
} else {
idx.StoppedCount++
group.Go(func() error {
// load instances
idx.Instances, err = dis.allinstances(r)
if err != nil {
return err
}
}
idx.TotalCount = len(idx.Instances)
// count how many are running and how many are stopped
for _, i := range idx.Instances {
if i.Running {
idx.RunningCount++
} else {
idx.StoppedCount++
}
}
idx.TotalCount = len(idx.Instances)
return nil
})
// get the log entries
group.Go(func() (err error) {
idx.Backups, err = dis.Instances.SnapshotLogFor("")
return
})
// get the static properties
idx.Config = dis.Config
@ -91,6 +106,9 @@ func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
// current time
idx.Time = time.Now().UTC()
// wait for everything!
group.Wait()
return
}

73
internal/component/instances/wisski_log.go Normal file
View file

@ -0,0 +1,73 @@
package instances
import (
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
)
// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug.
// An empty slug returns the log of backups.
func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snapshot, err error) {
snapshots, err = instances.SnapshotLog()
if err != nil {
return nil, err
}
return slicesx.Filter(snapshots, func(s models.Snapshot) bool {
return s.Slug == slug
}), nil
}
// SnapshotLog retrieves (and prunes) all entries in the snapshot log.
func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
// query the table!
table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
if err != nil {
return nil, err
}
// find all the snapshots
var snapshots []models.Snapshot
res := table.Find(&snapshots)
if res.Error != nil {
return nil, res.Error
}
// partition out the snapshots that have been deleted!
parts := slicesx.Partition(snapshots, func(s models.Snapshot) bool {
_, err := instances.Core.Environment.Stat(s.Path)
return !environment.IsNotExist(err)
})
// go and delete them!
if len(parts[false]) > 0 {
if err := table.Delete(parts[false]).Error; err != nil {
return nil, err
}
}
// return the ones that still exist
return parts[true], nil
}
// Snapshots returns the list of snapshots of this WissKI
func (wisski *WissKI) Snapshots() (snapshots []models.Snapshot, err error) {
return wisski.instances.SnapshotLogFor(wisski.Slug)
}
// AddSnapshotLog adds a log entry for the provided entry
func (instances *Instances) AddSnapshotLog(snapshot models.Snapshot) error {
// find the table
table, err := instances.SQL.QueryTable(false, models.SnapshotTable)
if err != nil {
return err
}
// and save it!
res := table.Create(&snapshot)
if res.Error != nil {
return res.Error
}
return nil
}

8
internal/component/instances/wisski_status.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
)
@ -20,6 +21,9 @@ type WissKIInfo struct {
Running bool
LastRebuild time.Time
// List of backups made
Snapshots []models.Snapshot
// WissKI content information
Prefixes []string // list of prefixes
Pathbuilders map[string]string // all the pathbuilders
@ -59,6 +63,10 @@ func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
info.Prefixes, _ = wisski.Prefixes()
return nil
})
group.Go(func() error {
info.Snapshots, _ = wisski.Snapshots()
return nil
})
}
err = group.Wait()

1
internal/component/snapshots/backup.go
View file

@ -44,7 +44,6 @@ type Backup struct {
// BackupDescription provides a description for a backup
type BackupDescription struct {
Dest string // Destination path
Auto bool // Was the path created automatically?
ConcurrentSnapshots int // maximum number of concurrent snapshots
}

17
internal/component/snapshots/log.go Normal file
View file

@ -0,0 +1,17 @@
package snapshots
import "github.com/FAU-CDI/wisski-distillery/internal/models"
func (backup *Backup) LogEntry() models.Snapshot {
return models.Snapshot{
Created: backup.StartTime,
Slug: "",
}
}
func (snapshot *Snapshot) LogEntry() models.Snapshot {
return models.Snapshot{
Created: snapshot.StartTime,
Slug: snapshot.Instance.Slug,
}
}

6
internal/component/snapshots/manager.go
View file

@ -7,6 +7,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
@ -15,6 +16,8 @@ import (
// Manager manages snapshots and backups
type Manager struct {
component.ComponentBase
SQL *sql.SQL
Instances *instances.Instances
Snapshotable []component.Snapshotable
@ -54,7 +57,7 @@ func (dis *Manager) NewArchivePath(prefix string) (path string) {
// newSnapshot name returns a new basename for a snapshot with the provided prefix.
// The name is guaranteed to be unique within this process.
func (*Manager) newSnapshotName(prefix string) string {
suffix, _ := password.Password(64) // silently ignore any errors!
suffix, _ := password.Password(10) // silently ignore any errors!
if prefix == "" {
prefix = "backup"
} else {
@ -68,7 +71,6 @@ func (*Manager) newSnapshotName(prefix string) string {
func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
for path == "" || environment.IsExist(err) {
path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
fmt.Println("path =>", prefix, "err => ", err)
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)
}
if err != nil {

25
internal/dis/backup_prune.go → internal/component/snapshots/prune.go
View file

@ -1,4 +1,4 @@
package dis
package snapshots
import (
"path/filepath"
@ -8,16 +8,16 @@ import (
)
// ShouldPrune determines if a file with the provided modtime
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
func (manager *Manager) ShouldPrune(modtime time.Time) bool {
return time.Since(modtime) > time.Duration(manager.Config.MaxBackupAge)*24*time.Hour
}
// PruneBackups prunes all backups older than the maximum backup age
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.SnapshotManager().ArchivePath()
// Prune prunes all backups and snapshots older than the maximum backup age
func (manager *Manager) PruneBackups(io stream.IOStream) error {
sPath := manager.ArchivePath()
// list all the files
entries, err := dis.Core.Environment.ReadDir(sPath)
entries, err := manager.Core.Environment.ReadDir(sPath)
if err != nil {
return err
}
@ -35,17 +35,20 @@ func (dis *Distillery) PruneBackups(io stream.IOStream) error {
}
// check if it should be pruned!
if !dis.ShouldPrune(info.ModTime()) {
if !manager.ShouldPrune(info.ModTime()) {
continue
}
// assemble path, and then remove the file!
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
io.Printf("Removing %s cause it is older than %d days", path, manager.Config.MaxBackupAge)
if err := dis.Core.Environment.Remove(path); err != nil {
if err := manager.Core.Environment.Remove(path); err != nil {
return err
}
}
return nil
// prune the snapshot log!
_, err = manager.Instances.SnapshotLog()
return err
}

1
internal/component/sql/connect.go
View file

@ -90,6 +90,7 @@ func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
// WaitQueryTable waits for a connection to succeed via QueryTable
func (sql *SQL) WaitQueryTable() error {
// TODO: Establish a convention on when to wait for this!
n := stream.FromDebug()
return wait.Wait(func() bool {
_, err := sql.QueryTable(true, models.InstanceTable)

5
internal/component/sql/update.go
View file

@ -91,6 +91,11 @@ func (sql *SQL) Update(io stream.IOStream) error {
&models.Metadatum{},
models.MetadataTable,
},
{
"snapshot",
&models.Snapshot{},
models.SnapshotTable,
},
}
// migrate all of the tables!

1
internal/dis/component.go
View file

@ -73,6 +73,7 @@ func (dis *Distillery) cInstances(thread int32) *instances.Instances {
func (dis *Distillery) cSnapshotManager(thread int32) *snapshots.Manager {
return component.PutComponent(&dis.pool, thread, dis.Core, func(snapshots *snapshots.Manager, thread int32) {
snapshots.SQL = dis.cSQL(thread)
snapshots.Instances = dis.cInstances(thread)
snapshots.Snapshotable = dis.cSnapshotable(thread)
snapshots.Backupable = dis.cBackupable(thread)

18
internal/models/snapshot.go Normal file
View file

@ -0,0 +1,18 @@
package models
import "time"
// SnapshotTable is the name of the table the [SnapshotLog] model is stored in
const SnapshotTable = "snapshot"
// Snapshot represents an entry in the snapshot log
type Snapshot struct {
Pk uint `gorm:"column:pk;primaryKey"`
Slug string `gorm:"column:slug"` // slug of instance
Created time.Time `gorm:"column:created"` // time the backup was created
Path string `gorm:"column:path;not null"` // path the backup is stored at
Packed bool `gorm:"column:packed;not null"` // was the backup packed, or was it staging only?
}

10
pkg/slicesx/slicesx.go
View file

@ -26,6 +26,16 @@ func Filter[T any](values []T, filter func(T) bool) []T {
return results
}
// Partition partitions values in T by the given functions.
func Partition[T any, P comparable](values []T, partition func(value T) P) map[P][]T {
result := make(map[P][]T)
for _, v := range values {
part := partition(v)
result[part] = append(result[part], v)
}
return result
}
// FilterClone is like [Filter], but creates a new slice
func FilterClone[T any](values []T, filter func(T) bool) (results []T) {
for _, value := range values {