'wdcli backup': Rework backup process
This commit reworks the backup process to dynamically find the list of components.
This commit is contained in:
Tom Wiesing
parent
55bee7422d
commit
5cd5ae9be2
32 changed files with 361 additions and 279 deletions
|
|
@ -11,11 +11,9 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
|
@ -36,13 +34,14 @@ type Backup struct {
|
|||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
// errors for the various components
|
||||
ComponentErrors map[string]error
|
||||
|
||||
// SQL and triplestore errors
|
||||
SQLErr error
|
||||
TSErr error
|
||||
TSErr error
|
||||
|
||||
// TODO: Make this proper
|
||||
ConfigFileErr error
|
||||
ConfigFilesManifest map[string]error
|
||||
ConfigFileErr error
|
||||
|
||||
// Snapshots containing instances
|
||||
InstanceListErr error
|
||||
|
|
@ -76,17 +75,13 @@ func (backup Backup) Report(w io.Writer) (int, error) {
|
|||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Errors =======\n")
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "SQLErr: %s\n", backup.SQLErr)
|
||||
fmt.Fprintf(cw, "TSErr: %s\n", backup.TSErr)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Config Files =======\n")
|
||||
encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
|
||||
|
||||
io.WriteString(cw, "======= Snapshots =======\n")
|
||||
for _, s := range backup.InstanceSnapshots {
|
||||
io.WriteString(cw, s.String())
|
||||
|
|
@ -103,6 +98,8 @@ func (backup Backup) Report(w io.Writer) (int, error) {
|
|||
return cw.Sum()
|
||||
}
|
||||
|
||||
// Backup makes a makes of the entire distillery.
|
||||
// To make a backup, all [BackupComponents] will be invoked.
|
||||
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
|
|
@ -123,75 +120,36 @@ func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription)
|
|||
return
|
||||
}
|
||||
|
||||
var errBackupSkipFile = errors.New("<file not found>")
|
||||
type backupResult struct {
|
||||
name string
|
||||
err error
|
||||
}
|
||||
|
||||
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
|
||||
// create a wait group, and message channel
|
||||
wg := &sync.WaitGroup{}
|
||||
files := make(chan string, 4)
|
||||
|
||||
// backup the sql
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
backups := dis.Backupable()
|
||||
|
||||
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
|
||||
files <- sqlPath
|
||||
files := make(chan string, len(backups)) // channel for files being added into the backups
|
||||
results := make(chan backupResult, len(backups)) // channel for results to be stored into
|
||||
backup.ComponentErrors = make(map[string]error, len(backups))
|
||||
|
||||
sql, err := os.Create(sqlPath)
|
||||
if err != nil {
|
||||
backup.SQLErr = err
|
||||
return
|
||||
}
|
||||
defer sql.Close()
|
||||
wg := &sync.WaitGroup{} // to wait for the results
|
||||
wg.Add(len(backups))
|
||||
for _, bc := range backups {
|
||||
go func(bc component.Backupable) {
|
||||
defer wg.Done()
|
||||
|
||||
// directly store the result
|
||||
backup.SQLErr = dis.SQL().BackupAll(io, sql)
|
||||
}()
|
||||
// find the backup destination
|
||||
dest := filepath.Join(backup.Description.Dest, bc.BackupName())
|
||||
files <- dest
|
||||
|
||||
// backup the triplestore
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
|
||||
files <- tsPath
|
||||
|
||||
// directly store the result
|
||||
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
|
||||
}()
|
||||
|
||||
// backup configuration files
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
|
||||
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
|
||||
backup.ConfigFileErr = err
|
||||
return
|
||||
}
|
||||
|
||||
configs := []string{
|
||||
dis.Config.ConfigPath,
|
||||
filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
|
||||
dis.Config.SelfOverridesFile,
|
||||
dis.Config.GlobalAuthorizedKeysFile,
|
||||
}
|
||||
|
||||
backup.ConfigFilesManifest = make(map[string]error, len(configs))
|
||||
for _, src := range configs {
|
||||
if !fsx.IsFile(src) {
|
||||
backup.ConfigFilesManifest[src] = errBackupSkipFile
|
||||
continue
|
||||
// make the backup and send the result!
|
||||
results <- backupResult{
|
||||
name: bc.Name(),
|
||||
err: bc.Backup(io, dest),
|
||||
}
|
||||
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
|
||||
|
||||
// copy the config file and store the error message
|
||||
files <- src
|
||||
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
|
||||
}
|
||||
}()
|
||||
}(bc)
|
||||
}
|
||||
|
||||
// backup instances
|
||||
wg.Add(1)
|
||||
|
|
@ -230,10 +188,18 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
|
|||
|
||||
}()
|
||||
|
||||
// wait for the group, then close the message channel.
|
||||
// finish processing all the results as soon as the group is done.
|
||||
go func() {
|
||||
defer close(results)
|
||||
wg.Wait()
|
||||
close(files)
|
||||
}()
|
||||
|
||||
// finish the message processing once results are finished.
|
||||
go func() {
|
||||
defer close(files) // no more file processing!
|
||||
for result := range results {
|
||||
backup.ComponentErrors[result.name] = result.err
|
||||
}
|
||||
}()
|
||||
|
||||
for file := range files {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/control"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
|
||||
|
|
@ -23,11 +23,11 @@ import (
|
|||
type components struct {
|
||||
|
||||
// installable components
|
||||
web lazy.Lazy[*web.Web]
|
||||
dis lazy.Lazy[*dis.Dis]
|
||||
ssh lazy.Lazy[*ssh.SSH]
|
||||
ts lazy.Lazy[*triplestore.Triplestore]
|
||||
sql lazy.Lazy[*sql.SQL]
|
||||
web lazy.Lazy[*web.Web]
|
||||
control lazy.Lazy[*control.Control]
|
||||
ssh lazy.Lazy[*ssh.SSH]
|
||||
ts lazy.Lazy[*triplestore.Triplestore]
|
||||
sql lazy.Lazy[*sql.SQL]
|
||||
|
||||
// other components
|
||||
instances lazy.Lazy[*instances.Instances]
|
||||
|
|
@ -67,23 +67,48 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
|
|||
})
|
||||
}
|
||||
|
||||
// Components returns all components that have a stack function
|
||||
func (dis *Distillery) Components() []component.InstallableComponent {
|
||||
return []component.InstallableComponent{
|
||||
func (dis *Distillery) ComponentsX() []component.Component {
|
||||
return []component.Component{
|
||||
dis.Web(),
|
||||
dis.Dis(),
|
||||
dis.SSH(),
|
||||
dis.Triplestore(),
|
||||
dis.SQL(),
|
||||
dis.Instances(),
|
||||
}
|
||||
}
|
||||
|
||||
// Backupable returns all the components that can be backuped up.
|
||||
func (dis *Distillery) Backupable() []component.Backupable {
|
||||
return getComponents[component.Backupable](dis)
|
||||
}
|
||||
|
||||
// Installables returns all components that can be installed
|
||||
func (dis *Distillery) Installables() []component.Installable {
|
||||
return getComponents[component.Installable](dis)
|
||||
}
|
||||
|
||||
func getComponents[C component.Component](dis *Distillery) (result []C) {
|
||||
all := dis.ComponentsX()
|
||||
|
||||
result = make([]C, 0, len(all))
|
||||
for _, c := range all {
|
||||
sc, ok := c.(C)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
result = append(result, sc)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (dis *Distillery) Web() *web.Web {
|
||||
return makeComponent(dis, &dis.components.web, nil)
|
||||
}
|
||||
|
||||
func (d *Distillery) Dis() *dis.Dis {
|
||||
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
|
||||
func (d *Distillery) Dis() *control.Control {
|
||||
return makeComponent(d, &d.components.control, func(ddis *control.Control) {
|
||||
ddis.ResolverFile = core.PrefixConfig
|
||||
ddis.Instances = d.Instances()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
|
|||
defer nquads.Close()
|
||||
|
||||
// directly store the result
|
||||
_, err = dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
|
||||
_, err = dis.Triplestore().Snapshot(nquads, instance.GraphDBRepository)
|
||||
return err
|
||||
}, &snapshot.ErrTriplestore)
|
||||
|
||||
|
|
@ -260,7 +260,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
|
|||
defer sql.Close()
|
||||
|
||||
// directly store the result
|
||||
return dis.SQL().Backup(io, sql, instance.SqlDatabase)
|
||||
return dis.SQL().Snapshot(io, sql, instance.SqlDatabase)
|
||||
}, &snapshot.ErrSQL)
|
||||
|
||||
// wait for the group!
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue