Rename packages
This commit is contained in:
Tom Wiesing
parent
49b8760527
commit
ef1243ea39
47 changed files with 524 additions and 369 deletions
324
internal/env/backup.go
vendored
324
internal/env/backup.go
vendored
|
|
@ -1,324 +0,0 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
// backupDescription is a description for a backup
|
||||
type BackupDescription struct {
|
||||
Dest string // destination path
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Backup struct {
|
||||
Description BackupDescription
|
||||
|
||||
// Start and End Time of the backup
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
// SQL and triplestore errors
|
||||
SQLErr error
|
||||
TSErr error
|
||||
|
||||
// TODO: Make this proper
|
||||
ConfigFileErr error
|
||||
ConfigFilesManifest map[string]error
|
||||
|
||||
// Snapshots containing instances
|
||||
InstanceListErr error
|
||||
InstanceSnapshots []Snapshot
|
||||
|
||||
// List of files included
|
||||
Manifest []string
|
||||
}
|
||||
|
||||
func (backup Backup) String() string {
|
||||
var builder strings.Builder
|
||||
backup.Report(&builder)
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Report writes a report from backup into w
|
||||
func (backup Backup) Report(w io.Writer) (int, error) {
|
||||
cw := countwriter.NewCountWriter(w)
|
||||
|
||||
encoder := json.NewEncoder(cw)
|
||||
encoder.SetIndent("", " ")
|
||||
|
||||
io.WriteString(cw, "======= Backup =======\n")
|
||||
|
||||
fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
|
||||
fmt.Fprintf(cw, "End: %s\n", backup.EndTime)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Description =======\n")
|
||||
encoder.Encode(backup.Description)
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Errors =======\n")
|
||||
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
|
||||
fmt.Fprintf(cw, "SQLErr: %s\n", backup.SQLErr)
|
||||
fmt.Fprintf(cw, "TSErr: %s\n", backup.TSErr)
|
||||
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
|
||||
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
io.WriteString(cw, "======= Config Files =======\n")
|
||||
encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
|
||||
|
||||
io.WriteString(cw, "======= Snapshots =======\n")
|
||||
for _, s := range backup.InstanceSnapshots {
|
||||
io.WriteString(cw, s.String())
|
||||
io.WriteString(cw, "\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "======= Manifest =======\n")
|
||||
for _, file := range backup.Manifest {
|
||||
io.WriteString(cw, file+"\n")
|
||||
}
|
||||
|
||||
io.WriteString(cw, "\n")
|
||||
|
||||
return cw.Sum()
|
||||
}
|
||||
|
||||
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
backup.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// do the create keeping track of time!
|
||||
logging.LogOperation(func() error {
|
||||
backup.StartTime = time.Now()
|
||||
backup.run(io, dis)
|
||||
backup.EndTime = time.Now()
|
||||
|
||||
return nil
|
||||
}, io, "Writing backup files")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var errBackupSkipFile = errors.New("<file not found>")
|
||||
|
||||
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
|
||||
// create a wait group, and message channel
|
||||
wg := &sync.WaitGroup{}
|
||||
files := make(chan string, 4)
|
||||
|
||||
// backup the sql
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
|
||||
files <- sqlPath
|
||||
|
||||
sql, err := os.Create(sqlPath)
|
||||
if err != nil {
|
||||
backup.SQLErr = err
|
||||
return
|
||||
}
|
||||
defer sql.Close()
|
||||
|
||||
// directly store the result
|
||||
backup.SQLErr = dis.SQL().BackupAll(io, sql)
|
||||
}()
|
||||
|
||||
// backup the triplestore
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
|
||||
files <- tsPath
|
||||
|
||||
// directly store the result
|
||||
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
|
||||
}()
|
||||
|
||||
// backup configuration files
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
|
||||
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
|
||||
backup.ConfigFileErr = err
|
||||
return
|
||||
}
|
||||
|
||||
configs := []string{
|
||||
dis.Config.ConfigPath,
|
||||
filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
|
||||
dis.Config.SelfOverridesFile,
|
||||
dis.Config.GlobalAuthorizedKeysFile,
|
||||
}
|
||||
|
||||
backup.ConfigFilesManifest = make(map[string]error, len(configs))
|
||||
for _, src := range configs {
|
||||
if !fsx.IsFile(src) {
|
||||
backup.ConfigFilesManifest[src] = errBackupSkipFile
|
||||
continue
|
||||
}
|
||||
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
|
||||
|
||||
// copy the config file and store the error message
|
||||
files <- src
|
||||
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
|
||||
}
|
||||
}()
|
||||
|
||||
// backup instances
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
|
||||
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
// list all instances
|
||||
instances, err := dis.AllInstances()
|
||||
if err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
|
||||
|
||||
backup.InstanceSnapshots = make([]Snapshot, len(instances))
|
||||
for i, instance := range instances {
|
||||
backup.InstanceSnapshots[i] = func() Snapshot {
|
||||
dir := filepath.Join(instancesBackupDir, instance.Slug)
|
||||
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
|
||||
return Snapshot{
|
||||
ErrPanic: err,
|
||||
}
|
||||
}
|
||||
|
||||
files <- dir
|
||||
return instance.Snapshot(iochild, SnapshotDescription{
|
||||
Dest: dir,
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
// wait for the group, then close the message channel.
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(files)
|
||||
}()
|
||||
|
||||
for file := range files {
|
||||
// get the relative path to the root of the manifest.
|
||||
// nothing *should* go wrong, but in case it does, use the original path.
|
||||
path, err := filepath.Rel(backup.Description.Dest, file)
|
||||
if err != nil {
|
||||
path = file
|
||||
}
|
||||
|
||||
// write it to the command line
|
||||
// and also add it to the manifest
|
||||
io.Printf("\033[2K\r%s", path)
|
||||
backup.Manifest = append(backup.Manifest, path)
|
||||
}
|
||||
slices.Sort(backup.Manifest) // backup the manifest
|
||||
io.Println("")
|
||||
|
||||
// sort the instances manifest
|
||||
slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
|
||||
return a.Instance.Slug < b.Instance.Slug
|
||||
})
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this backup.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (backup Backup) WriteReport(io stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
|
||||
io.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := os.Create(reportPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = report.WriteString(backup.String())
|
||||
return err
|
||||
}, io, "Writing backup report")
|
||||
}
|
||||
|
||||
// ShouldPrune determines if a file with the provided modtime
|
||||
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
|
||||
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
|
||||
}
|
||||
|
||||
// PruneBackups prunes all backups older than the maximum backup age
|
||||
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
|
||||
sPath := dis.SnapshotsArchivePath()
|
||||
|
||||
// list all the files
|
||||
entries, err := os.ReadDir(sPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
// skip directories
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// grab info about the file
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if it should be pruned!
|
||||
if !dis.ShouldPrune(info.ModTime()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// assemble path, and then remove the file!
|
||||
path := filepath.Join(sPath, entry.Name())
|
||||
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue