package env import ( "encoding/json" "fmt" "io" "io/fs" "os" "path/filepath" "strings" "sync" "time" "github.com/FAU-CDI/wisski-distillery/internal/core" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/pkg/errors" "github.com/tkw1536/goprogram/stream" "golang.org/x/exp/slices" ) // backupDescription is a description for a backup type BackupDescription struct { Dest string // destination path } // Snapshot represents the result of generating a snapshot type Backup struct { Description BackupDescription // Start and End Time of the backup StartTime time.Time EndTime time.Time // various error states, which are ignored when creating the snapshot ErrPanic interface{} // SQL and triplestore errors SQLErr error TSErr error // TODO: Make this proper ConfigFileErr error ConfigFilesManifest map[string]error // Snapshots containing instances InstanceListErr error InstanceSnapshots []Snapshot // List of files included Manifest []string } func (backup Backup) String() string { var builder strings.Builder backup.Report(&builder) return builder.String() } // Report writes a report from backup into w func (backup Backup) Report(w io.Writer) { // TODO: Errors encoder := json.NewEncoder(w) encoder.SetIndent("", " ") io.WriteString(w, "======= Backup =======\n") fmt.Fprintf(w, "Start: %s\n", backup.StartTime) fmt.Fprintf(w, "End: %s\n", backup.EndTime) io.WriteString(w, "\n") io.WriteString(w, "======= Description =======\n") encoder.Encode(backup.Description) io.WriteString(w, "\n") io.WriteString(w, "======= Errors =======\n") fmt.Fprintf(w, "Panic: %v\n", backup.ErrPanic) fmt.Fprintf(w, "SQLErr: %s\n", backup.SQLErr) fmt.Fprintf(w, "TSErr: %s\n", backup.TSErr) fmt.Fprintf(w, "ConfigFileErr: %s\n", backup.ConfigFileErr) fmt.Fprintf(w, "InstanceListErr: %s\n", backup.InstanceListErr) io.WriteString(w, "\n") io.WriteString(w, "======= Config Files =======\n") encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest io.WriteString(w, "======= Snapshots =======\n") for _, s := range backup.InstanceSnapshots { io.WriteString(w, s.String()) io.WriteString(w, "\n") } io.WriteString(w, "======= Manifest =======\n") for _, file := range backup.Manifest { io.WriteString(w, file+"\n") } io.WriteString(w, "\n") } func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) { backup.Description = description // catch anything critical that happened during the snapshot defer func() { backup.ErrPanic = recover() }() // do the create keeping track of time! logging.LogOperation(func() error { backup.StartTime = time.Now() backup.run(io, dis) backup.EndTime = time.Now() return nil }, io, "Writing backup files") return } var errBackupSkipFile = errors.New("") func (backup *Backup) run(io stream.IOStream, dis *Distillery) { // create a wait group, and message channel wg := &sync.WaitGroup{} files := make(chan string, 4) // backup the sql wg.Add(1) go func() { defer wg.Done() sqlPath := filepath.Join(backup.Description.Dest, "sql.sql") files <- sqlPath sql, err := os.Create(sqlPath) if err != nil { backup.SQLErr = err return } defer sql.Close() // directly store the result backup.SQLErr = dis.SQL().BackupAll(io, sql) }() // backup the triplestore wg.Add(1) go func() { defer wg.Done() tsPath := filepath.Join(backup.Description.Dest, "triplestore") files <- tsPath // directly store the result backup.TSErr = dis.Triplestore().BackupAll(tsPath) }() // backup configuration files wg.Add(1) go func() { defer wg.Done() cfgBackupDir := filepath.Join(backup.Description.Dest, "config") if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil { backup.ConfigFileErr = err return } configs := []string{ dis.Config.ConfigPath, filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable dis.Config.SelfOverridesFile, dis.Config.GlobalAuthorizedKeysFile, } backup.ConfigFilesManifest = make(map[string]error, len(configs)) for _, src := range configs { if !fsx.IsFile(src) { backup.ConfigFilesManifest[src] = errBackupSkipFile continue } dest := filepath.Join(cfgBackupDir, filepath.Base(src)) // copy the config file and store the error message files <- src backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src) } }() // backup instances wg.Add(1) go func() { defer wg.Done() instancesBackupDir := filepath.Join(backup.Description.Dest, "instances") if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil { backup.InstanceListErr = err return } // list all instances instances, err := dis.AllInstances() if err != nil { backup.InstanceListErr = err return } iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0) backup.InstanceSnapshots = make([]Snapshot, len(instances)) for i, instance := range instances { backup.InstanceSnapshots[i] = func() Snapshot { dir := filepath.Join(instancesBackupDir, instance.Slug) if err := os.Mkdir(dir, fs.ModeDir); err != nil { return Snapshot{ ErrPanic: err, } } files <- dir return instance.Snapshot(iochild, SnapshotDescription{ Dest: dir, }) }() } }() // wait for the group, then close the message channel. go func() { wg.Wait() close(files) }() for file := range files { // get the relative path to the root of the manifest. // nothing *should* go wrong, but in case it does, use the original path. path, err := filepath.Rel(backup.Description.Dest, file) if err != nil { path = file } // write it to the command line // and also add it to the manifest io.Printf("\033[2K\r%s", path) backup.Manifest = append(backup.Manifest, path) } slices.Sort(backup.Manifest) // backup the manifest io.Println("") // sort the instances manifest slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool { return a.Instance.Slug < b.Instance.Slug }) } // WriteReport writes out the report belonging to this backup. // It is a separate function, to allow writing it indepenently of the rest. func (backup Backup) WriteReport(io stream.IOStream) error { return logging.LogOperation(func() error { reportPath := filepath.Join(backup.Description.Dest, "report.txt") io.Println(reportPath) // create the report file! report, err := os.Create(reportPath) if err != nil { return err } defer report.Close() // print the report into it! _, err = report.WriteString(backup.String()) return err }, io, "Writing backup report") } // ShouldPrune determines if a file with the provided modtime func (dis *Distillery) ShouldPrune(modtime time.Time) bool { return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour } // PruneBackups prunes all backups older than the maximum backup age func (dis *Distillery) PruneBackups(io stream.IOStream) error { sPath := dis.SnapshotsArchivePath() // list all the files entries, err := os.ReadDir(sPath) if err != nil { return err } for _, entry := range entries { // skip directories if entry.IsDir() { continue } // grab info about the file info, err := entry.Info() if err != nil { return err } // check if it should be pruned! if !dis.ShouldPrune(info.ModTime()) { continue } // assemble path, and then remove the file! path := filepath.Join(sPath, entry.Name()) io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge) if err := os.Remove(path); err != nil { return err } } return nil }