rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

snapshots: Prepare for restructuring

Browse source 
This commit renames the 'wisski' package to 'dis' and prepares the
snapshots component for restructuring.
This commit is contained in:
Tom Wiesing 2022-10-01 19:53:18 +02:00
parent f58920baf4
commit 1dac09bc03
No known key found for this signature in database
12 changed files with 62 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cmd/backup.go
View file

@ -53,7 +53,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
if !bk.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
sPath, err = dis.Snapshots().NewStagingDir("")
sPath, err = dis.SnapshotManager().NewStagingDir("")
if err != nil {
return errSnapshotFailed.Wrap(err)
}
@ -65,7 +65,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
// staging mode: use dest as a destination
sPath = bk.Positionals.Dest
if sPath == "" {
sPath, err = dis.Snapshots().NewStagingDir("")
sPath, err = dis.SnapshotManager().NewStagingDir("")
if err != nil {
return errSnapshotFailed.Wrap(err)
}
@ -100,7 +100,7 @@ func (bk backupC) Run(context wisski_distillery.Context) error {
// create the archive path
archivePath := bk.Positionals.Dest
if archivePath == "" {
archivePath = dis.Snapshots().NewArchivePath("")
archivePath = dis.SnapshotManager().NewArchivePath("")
}
// and write everything into it!

10
cmd/snapshot.go
View file

@ -4,8 +4,8 @@ import (
"io/fs"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
@ -55,7 +55,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
if !bi.StagingOnly {
// regular mode: create a temporary staging directory
logging.LogMessage(context.IOStream, "Creating new snapshot staging directory")
sPath, err = dis.Snapshots().NewStagingDir(instance.Slug)
sPath, err = dis.SnapshotManager().NewStagingDir(instance.Slug)
if err != nil {
return errSnapshotFailed.Wrap(err)
}
@ -67,7 +67,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// staging mode: use dest as a destination
sPath = bi.Positionals.Dest
if sPath == "" {
sPath, err = dis.Snapshots().NewStagingDir(instance.Slug)
sPath, err = dis.SnapshotManager().NewStagingDir(instance.Slug)
if err != nil {
return errSnapshotFailed.Wrap(err)
}
@ -87,7 +87,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// take a snapshot into the staging area!
logging.LogOperation(func() error {
sreport := dis.Snapshot(instance, context.IOStream, wisski.SnapshotDescription{
sreport := dis.SnapshotManager().NewSnapshot(instance, context.IOStream, snapshots.SnapshotDescription{
Dest: sPath,
Keepalive: bi.Keepalive,
})
@ -107,7 +107,7 @@ func (bi snapshot) Run(context wisski_distillery.Context) error {
// create the archive path
archivePath := bi.Positionals.Dest
if archivePath == "" {
archivePath = dis.Snapshots().NewArchivePath(instance.Slug)
archivePath = dis.SnapshotManager().NewArchivePath(instance.Slug)
}
// and write everything into it!

4
cmd/system_update.go
View file

@ -70,8 +70,8 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
for _, d := range []string{
dis.Config.DeployRoot,
dis.Instances().Path(),
dis.Snapshots().StagingPath(),
dis.Snapshots().ArchivePath(),
dis.SnapshotManager().StagingPath(),
dis.SnapshotManager().ArchivePath(),
} {
context.Println(d)
if err := dis.Core.Environment.MkdirAll(d, environment.DefaultDirPerm); err != nil {

19
internal/backup/backup.go
View file

@ -9,7 +9,8 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/internal/dis"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/status"
@ -18,7 +19,7 @@ import (
)
// New create a new Backup
func New(io stream.IOStream, dis *wisski.Distillery, description Description) (backup Backup) {
func New(io stream.IOStream, dis *dis.Distillery, description Description) (backup Backup) {
backup.Description = description
// catch anything critical that happened during the snapshot
@ -38,7 +39,7 @@ func New(io stream.IOStream, dis *wisski.Distillery, description Description) (b
return
}
func (backup *Backup) run(ios stream.IOStream, dis *wisski.Distillery) {
func (backup *Backup) run(ios stream.IOStream, dis *dis.Distillery) {
//
// MANIFEST
//
@ -120,27 +121,27 @@ func (backup *Backup) run(ios stream.IOStream, dis *wisski.Distillery) {
}
// re-use the backup of the snapshots
backup.InstanceSnapshots = status.Group[instances.WissKI, wisski.Snapshot]{
backup.InstanceSnapshots = status.Group[instances.WissKI, snapshots.Snapshot]{
PrefixString: func(item instances.WissKI, index int) string {
return fmt.Sprintf("[snapshot %s]: ", item.Slug)
},
PrefixAlign: true,
Handler: func(instance instances.WissKI, index int, writer io.Writer) wisski.Snapshot {
Handler: func(instance instances.WissKI, index int, writer io.Writer) snapshots.Snapshot {
dir := filepath.Join(instancesBackupDir, instance.Slug)
if err := dis.Core.Environment.Mkdir(dir, environment.DefaultDirPerm); err != nil {
return wisski.Snapshot{
return snapshots.Snapshot{
ErrPanic: err,
}
}
manifest <- dir
return dis.Snapshot(instance, stream.NewIOStream(writer, writer, nil, 0), wisski.SnapshotDescription{
return dis.SnapshotManager().NewSnapshot(instance, stream.NewIOStream(writer, writer, nil, 0), snapshots.SnapshotDescription{
Dest: dir,
})
},
ResultString: func(res wisski.Snapshot, item instances.WissKI, index int) string {
ResultString: func(res snapshots.Snapshot, item instances.WissKI, index int) string {
return "done"
},
WaitString: status.DefaultWaitString[instances.WissKI],
@ -154,7 +155,7 @@ func (backup *Backup) run(ios stream.IOStream, dis *wisski.Distillery) {
<-manifestDone
// sort the instances manifest
slices.SortFunc(backup.InstanceSnapshots, func(a, b wisski.Snapshot) bool {
slices.SortFunc(backup.InstanceSnapshots, func(a, b snapshots.Snapshot) bool {
return a.Instance.Slug < b.Instance.Slug
})
}

4
internal/backup/report.go
View file

@ -8,7 +8,7 @@ import (
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/internal/component/snapshots"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
@ -42,7 +42,7 @@ type Backup struct {
// Snapshots containing instances
InstanceListErr error
InstanceSnapshots []wisski.Snapshot
InstanceSnapshots []snapshots.Snapshot
// List of files included
Manifest []string

26
internal/wisski/snapshot.go → internal/component/snapshots/snapshot_make.go
View file

@ -1,4 +1,4 @@
package wisski
package snapshots
import (
"encoding/json"
@ -109,7 +109,7 @@ func (snapshot Snapshot) Report(w io.Writer) (int, error) {
}
// Snapshot creates a new snapshot of this instance into dest
func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
func (snapshots *Manager) NewSnapshot(instance instances.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
// setup the snapshot
snapshot.Description = desc
snapshot.Instance = instance.Instance
@ -123,8 +123,8 @@ func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, d
logging.LogOperation(func() error {
snapshot.StartTime = time.Now().UTC()
snapshot.makeBlackbox(io, dis, instance)
snapshot.makeWhitebox(io, dis, instance)
snapshot.makeBlackbox(io, snapshots, instance)
snapshot.makeWhitebox(io, snapshots, instance)
snapshot.EndTime = time.Now().UTC()
return nil
@ -136,7 +136,7 @@ func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, d
// makeBlackbox runs the blackbox backup of the system.
// It pauses the Instance, if a consistent state is required.
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, instance instances.WissKI) {
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
stack := instance.Barrel()
og := opgroup.NewOpGroup[string](4)
@ -166,7 +166,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fmt.Fprintln(line, bkPath)
files <- bkPath
info, err := dis.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
info, err := snapshots.Core.Environment.Create(bkPath, environment.DefaultFilePerm)
if err != nil {
return err
}
@ -188,7 +188,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
// copy over whatever is in the base directory
defer fmt.Fprintln(line, "done")
return fsx.CopyDirectory(dis.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
return fsx.CopyDirectory(snapshots.Core.Environment, fsPath, instance.FilesystemBase, func(dst, src string) {
fmt.Fprintln(line, dst)
files <- dst
})
@ -205,14 +205,14 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fmt.Fprintln(line, tsPath)
files <- tsPath
nquads, err := dis.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
nquads, err := snapshots.Core.Environment.Create(tsPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer nquads.Close()
// directly store the result
_, err = dis.Triplestore().Snapshot(nquads, instance.GraphDBRepository)
_, err = snapshots.TS.Snapshot(nquads, instance.GraphDBRepository)
return err
}, &snapshot.ErrTriplestore)
@ -226,14 +226,14 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
fmt.Fprintln(line, sqlPath)
files <- sqlPath
sql, err := dis.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
sql, err := snapshots.Core.Environment.Create(sqlPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer sql.Close()
// directly store the result
return dis.SQL().Snapshot(io, sql, instance.SqlDatabase)
return snapshots.SQL.Snapshot(io, sql, instance.SqlDatabase)
}, &snapshot.ErrSQL)
// wait for the group!
@ -242,7 +242,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
// makeWhitebox runs the whitebox backup of the system.
// The instance should be running during this step.
func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, instance instances.WissKI) {
func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, snapshots *Manager, instance instances.WissKI) {
og := opgroup.NewOpGroup[string](1)
// write pathbuilders
@ -252,7 +252,7 @@ func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, dis *Distillery, inst
files <- pbPath
// create the directory!
if err := dis.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
if err := snapshots.Core.Environment.Mkdir(pbPath, environment.DefaultDirPerm); err != nil {
return err
}

23
internal/component/snapshots/snapshots.go
View file

@ -6,38 +6,43 @@ import (
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
)
// Snapshots manages snapshots and backups
type Snapshots struct {
// Manager manages snapshots and backups
type Manager struct {
component.ComponentBase
TS *triplestore.Triplestore
SQL *sql.SQL
}
func (Snapshots) Name() string { return "snapshots" }
func (Manager) Name() string { return "snapshots" }
// Path returns the path that contains all snapshot related data.
func (dis *Snapshots) Path() string {
func (dis *Manager) Path() string {
return filepath.Join(dis.Config.DeployRoot, "snapshots")
}
// StagingPath returns the path to the directory containing a temporary staging area for snapshots.
// Use NewSnapshotStagingDir to generate a new staging area.
func (dis *Snapshots) StagingPath() string {
func (dis *Manager) StagingPath() string {
return filepath.Join(dis.Path(), "staging")
}
// ArchivePath returns the path to the directory containing all exported archives.
// Use NewSnapshotArchivePath to generate a path to a new archive in this directory.
func (dis *Snapshots) ArchivePath() string {
func (dis *Manager) ArchivePath() string {
return filepath.Join(dis.Path(), "archives")
}
// NewArchivePath returns the path to a new archive with the provided prefix.
// The path is guaranteed to not exist.
func (dis *Snapshots) NewArchivePath(prefix string) (path string) {
func (dis *Manager) NewArchivePath(prefix string) (path string) {
// TODO: Consider moving these into a subdirectory with the provided prefix.
for path == "" || fsx.Exists(dis.Environment, path) {
name := dis.newSnapshotName(prefix) + ".tar.gz"
@ -48,7 +53,7 @@ func (dis *Snapshots) NewArchivePath(prefix string) (path string) {
// newSnapshot name returns a new basename for a snapshot with the provided prefix.
// The name is guaranteed to be unique within this process.
func (*Snapshots) newSnapshotName(prefix string) string {
func (*Manager) newSnapshotName(prefix string) string {
suffix, _ := password.Password(64) // silently ignore any errors!
if prefix == "" {
prefix = "backup"
@ -60,7 +65,7 @@ func (*Snapshots) newSnapshotName(prefix string) string {
// NewStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Snapshots) NewStagingDir(prefix string) (path string, err error) {
func (dis *Manager) NewStagingDir(prefix string) (path string, err error) {
for path == "" || environment.IsExist(err) {
path = filepath.Join(dis.StagingPath(), dis.newSnapshotName(prefix))
err = dis.Core.Environment.Mkdir(path, environment.DefaultFilePerm)

4
internal/wisski/backup_prune.go → internal/dis/backup_prune.go
View file

@ -1,4 +1,4 @@
package wisski
package dis
import (
"path/filepath"
@ -14,7 +14,7 @@ func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
// PruneBackups prunes all backups older than the maximum backup age
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.Snapshots().ArchivePath()
sPath := dis.SnapshotManager().ArchivePath()
// list all the files
entries, err := dis.Core.Environment.ReadDir(sPath)

11
internal/wisski/component.go → internal/dis/component.go
View file

@ -1,4 +1,4 @@
package wisski
package dis
import (
"time"
@ -30,7 +30,7 @@ type components struct {
// other components
instances lazy.Lazy[*instances.Instances]
snapshots lazy.Lazy[*snapshots.Snapshots]
snapshots lazy.Lazy[*snapshots.Manager]
}
//
@ -75,9 +75,10 @@ func (dis *Distillery) Instances() *instances.Instances {
})
}
func (dis *Distillery) Snapshots() *snapshots.Snapshots {
return component.Initialize(dis.Core, &dis.components.snapshots, func(snapshots *snapshots.Snapshots) {
func (dis *Distillery) SnapshotManager() *snapshots.Manager {
return component.Initialize(dis.Core, &dis.components.snapshots, func(snapshots *snapshots.Manager) {
snapshots.SQL = dis.SQL()
snapshots.TS = dis.Triplestore()
})
}

2
internal/wisski/distillery.go → internal/dis/distillery.go
View file

@ -1,4 +1,4 @@
package wisski
package dis
import (
"context"

2
internal/wisski/init.go → internal/dis/init.go
View file

@ -1,4 +1,4 @@
package wisski
package dis
import (
"github.com/FAU-CDI/wisski-distillery/internal/component"

6
program.go
View file

@ -4,14 +4,14 @@ import (
"os/user"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/internal/dis"
"github.com/tkw1536/goprogram"
"github.com/tkw1536/goprogram/exit"
)
// these define the ggman-specific program types
// none of these are strictly needed, they're just around for convenience
type wdcliEnv = *wisski.Distillery
type wdcliEnv = *dis.Distillery
type wdcliParameters = core.Params
type wdcliRequirements = core.Requirements
type wdCliFlags = core.Flags
@ -49,7 +49,7 @@ func NewProgram() Program {
},
NewEnvironment: func(params wdcliParameters, context Context) (e wdcliEnv, err error) {
return wisski.NewDistillery(params, context.Args.Flags, context.Description.Requirements)
return dis.NewDistillery(params, context.Args.Flags, context.Description.Requirements)
},
}
}