rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

'wdcli backup': Rework backup process

Browse source 
This commit reworks the backup process to dynamically find the list of
components.
This commit is contained in:
Tom Wiesing 2022-09-17 16:30:32 +02:00
parent 55bee7422d
commit 5cd5ae9be2
No known key found for this signature in database
32 changed files with 361 additions and 279 deletions
Download patch file Download diff file Expand all files Collapse all files

25
internal/component/component.go
View file

@ -3,6 +3,7 @@ package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/tkw1536/goprogram/stream"
)
// Component represents a logical subsystem of the distillery.
@ -31,20 +32,34 @@ type Component interface {
Base() *ComponentBase
}
// InstallableComponent implements an installable component
type InstallableComponent interface {
// Installable implements an installable component.
type Installable interface {
Component
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call
Stack() Installable
// This should internally call [ComponentBase.MakeStack]
Stack() StackWithResources
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
Context(parent InstallationContext) InstallationContext
}
// Backupable represents a component with a Backup method
type Backupable interface {
Component
// BackupName returns a new name to be used as an argument for path.
BackupName() string
// Backup backs up this component into the destination path path.
//
// The destination path may be a folder or directory, depending on the component.
// The destination path does not need to exist.
Backup(io stream.IOStream, path string) error
}
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
@ -67,7 +82,7 @@ func (ComponentBase) Context(parent InstallationContext) InstallationContext {
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack Installable) Installable {
func (cb ComponentBase) MakeStack(stack StackWithResources) StackWithResources {
stack.Dir = cb.Dir
return stack
}

48
internal/component/control/backup.go Normal file
View file

@ -0,0 +1,48 @@
package control
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/stream"
)
func (*Control) BackupName() string {
return "config"
}
// Backup backups all control plane configuration files into dest
func (control *Control) Backup(io stream.IOStream, dest string) error {
// create the destination directory, TODO: outsource this
if err := os.Mkdir(dest, fs.ModeDir); err != nil {
return err
}
files := control.backupFiles()
for _, src := range files {
dst := filepath.Join(dest, filepath.Base(src)) // destination path
// if the src file does not exist, don't copy it!
if !fsx.IsFile(src) { // TODO: log this somewhere
continue
}
if err := fsx.CopyFile(dst, src); err != nil {
return err
}
}
return nil
}
// backupfiles lists the files to be backed up.
func (control *Control) backupFiles() []string {
return []string{
control.Config.ConfigPath,
control.Config.ExecutablePath(),
control.Config.SelfOverridesFile,
control.Config.GlobalAuthorizedKeysFile,
}
}

0
internal/component/dis/dis.env → internal/component/control/control.env
View file

54
internal/component/control/control.go Normal file
View file

@ -0,0 +1,54 @@
package control
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
// Control represents the control server
type Control struct {
component.ComponentBase
Instances *instances.Instances
ResolverFile string
}
func (control Control) Name() string {
return "dis" // TODO: Rename this to control!
}
//go:embed all:control control.env
var resources embed.FS
func (control Control) Stack() component.StackWithResources {
return control.ComponentBase.MakeStack(component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": control.Config.DefaultHost(),
"LETSENCRYPT_HOST": control.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": control.Config.CertbotEmail,
"CONFIG_PATH": control.Config.ConfigPath,
"DEPLOY_ROOT": control.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": control.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": control.Config.SelfOverridesFile,
},
TouchFiles: []string{control.ResolverFile},
CopyContextFiles: []string{core.Executable},
})
}
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: control.Config.CurrentExecutable(),
}
}

0
internal/component/dis/stack/Dockerfile → internal/component/control/control/Dockerfile
View file

0
internal/component/dis/stack/docker-compose.yml → internal/component/control/control/docker-compose.yml
View file

0
internal/component/dis/html/index.html → internal/component/control/html/index.html
View file

0
internal/component/dis/html/instance.html → internal/component/control/html/instance.html
View file

0
internal/component/dis/html/static/autolink.css → internal/component/control/html/static/autolink.css
View file

0
internal/component/dis/html/static/autolink.js → internal/component/control/html/static/autolink.js
View file

0
internal/component/dis/html/static/dis.css → internal/component/control/html/static/dis.css
View file

28
internal/component/dis/info.go → internal/component/control/info.go
View file

@ -1,4 +1,4 @@
package dis
package control
import (
"embed"
@ -14,11 +14,9 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
_ "embed"
)
func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
func (control *Control) info(io stream.IOStream) (http.Handler, error) {
mux := http.NewServeMux()
// handle everything under /dis/!
@ -31,7 +29,7 @@ func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
})
// static stuff
static, err := dis.disStatic()
static, err := control.disStatic()
if err != nil {
return nil, err
}
@ -39,22 +37,22 @@ func (dis *Dis) info(io stream.IOStream) (http.Handler, error) {
// render everything
mux.Handle("/dis/index", httpx.HTMLHandler[disIndex]{
Handler: dis.disIndex,
Handler: control.disIndex,
Template: indexTemplate,
})
mux.Handle("/dis/instance/", httpx.HTMLHandler[disInstance]{
Handler: dis.disInstance,
Handler: control.disInstance,
Template: instanceTemplate,
})
// api -- for future usage
mux.Handle("/dis/api/v1/instance/get/", httpx.JSON(dis.getinstance))
mux.Handle("/dis/api/v1/instance/all", httpx.JSON(dis.allinstances))
mux.Handle("/dis/api/v1/instance/get/", httpx.JSON(control.getinstance))
mux.Handle("/dis/api/v1/instance/all", httpx.JSON(control.allinstances))
// ensure that everyone is logged in!
return httpx.BasicAuth(mux, "WissKI Distillery Admin", func(user, pass string) bool {
return user == dis.Config.DisAdminUser && pass == dis.Config.DisAdminPassword
return user == control.Config.DisAdminUser && pass == control.Config.DisAdminPassword
}), nil
}
@ -70,7 +68,7 @@ type disIndex struct {
StoppedCount int
}
func (dis *Dis) disIndex(r *http.Request) (idx disIndex, err error) {
func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
// load instances
idx.Instances, err = dis.allinstances(r)
if err != nil {
@ -104,7 +102,7 @@ type disInstance struct {
Info instances.Info
}
func (dis *Dis) disInstance(r *http.Request) (is disInstance, err error) {
func (dis *Control) disInstance(r *http.Request) (is disInstance, err error) {
// find the slug as the last component of path!
slug := strings.TrimSuffix(r.URL.Path, "/")
slug = slug[strings.LastIndex(slug, "/")+1:]
@ -134,7 +132,7 @@ func (dis *Dis) disInstance(r *http.Request) (is disInstance, err error) {
//go:embed html/static
var htmlStaticFS embed.FS
func (*Dis) disStatic() (http.Handler, error) {
func (*Control) disStatic() (http.Handler, error) {
fs, err := fs.Sub(htmlStaticFS, "html/static")
if err != nil {
return nil, err
@ -151,7 +149,7 @@ var indexTemplate = template.Must(template.New("index.html").Parse(indexTemplate
var instanceTemplateString string
var instanceTemplate = template.Must(template.New("instance.html").Parse(instanceTemplateString))
func (dis *Dis) getinstance(r *http.Request) (info instances.Info, err error) {
func (dis *Control) getinstance(r *http.Request) (info instances.Info, err error) {
// find the slug as the last component of path!
slug := strings.TrimSuffix(r.URL.Path, "/")
slug = slug[strings.LastIndex(slug, "/")+1:]
@ -169,7 +167,7 @@ func (dis *Dis) getinstance(r *http.Request) (info instances.Info, err error) {
return wisski.Info(false)
}
func (dis *Dis) allinstances(*http.Request) (infos []instances.Info, err error) {
func (dis *Control) allinstances(*http.Request) (infos []instances.Info, err error) {
var errgroup errgroup.Group
// list all the instances

14
internal/component/dis/resolver.go → internal/component/control/resolver.go
View file

@ -1,4 +1,4 @@
package dis
package control
import (
"fmt"
@ -11,11 +11,11 @@ import (
"github.com/tkw1536/goprogram/stream"
)
func (dis Dis) ResolverConfigPath() string {
return filepath.Join(dis.Dir, dis.ResolverFile)
func (control Control) ResolverConfigPath() string {
return filepath.Join(control.Dir, control.ResolverFile)
}
func (dis Dis) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
@ -23,20 +23,20 @@ func (dis Dis) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err err
}
// handle the default domain name!
domainName := dis.Config.DefaultDomain
domainName := control.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range dis.Config.SelfExtraDomains {
for _, domain := range control.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := dis.ResolverConfigPath()
prefixFile := control.ResolverConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {

24
internal/component/dis/self.go → internal/component/control/self.go
View file

@ -1,4 +1,4 @@
package dis
package control
import (
"encoding/json"
@ -11,10 +11,10 @@ import (
)
// self returns the handler for the self overrides
func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(dis.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", dis.Config.SelfOverridesFile)
overrides, err := os.Open(control.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
if err != nil {
return redirect, err
}
@ -28,10 +28,10 @@ func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
if redirect.Overrides == nil {
redirect.Overrides = make(map[string]string)
}
redirect.Overrides[""] = dis.Config.SelfRedirect.String()
redirect.Overrides[""] = control.Config.SelfRedirect.String()
// create a redirect server
redirect.Fallback, err = dis.selfFallback()
redirect.Fallback, err = control.selfFallback()
if err != nil {
return redirect, err
}
@ -42,24 +42,22 @@ func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
return redirect, nil
}
func (dis *Dis) selfFallback() (http.Handler, error) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
dis.serveFallback(w, r)
}), nil
func (control *Control) selfFallback() (http.Handler, error) {
return http.HandlerFunc(control.serveFallback), nil
}
var notFoundText = []byte("not found")
func (dis *Dis) serveFallback(w http.ResponseWriter, r *http.Request) {
func (control *Control) serveFallback(w http.ResponseWriter, r *http.Request) {
slug := dis.Config.SlugFromHost(r.Host)
slug := control.Config.SlugFromHost(r.Host)
if slug == "" {
w.WriteHeader(http.StatusNotFound)
w.Write(notFoundText)
return
}
if ok, _ := dis.Instances.Has(slug); !ok {
if ok, _ := control.Instances.Has(slug); !ok {
w.WriteHeader(http.StatusNotFound)
fmt.Fprintf(w, "WissKI %q not found\n", slug)
return

10
internal/component/dis/server.go → internal/component/control/server.go
View file

@ -1,4 +1,4 @@
package dis
package control
import (
"net/http"
@ -7,19 +7,19 @@ import (
)
// Server returns an http.Mux that implements the main server instance
func (dis Dis) Server(io stream.IOStream) (http.Handler, error) {
func (control Control) Server(io stream.IOStream) (http.Handler, error) {
// self server
self, err := dis.self(io)
self, err := control.self(io)
if err != nil {
return nil, err
}
resolver, err := dis.resolver(io)
resolver, err := control.resolver(io)
if err != nil {
return nil, err
}
info, err := dis.info(io)
info, err := control.info(io)
if err != nil {
return nil, err
}

53
internal/component/dis/dis.go
View file

@ -1,53 +0,0 @@
package dis
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
type Dis struct {
component.ComponentBase
Instances *instances.Instances
ResolverFile string
}
func (dis Dis) Name() string {
return "dis"
}
//go:embed all:stack dis.env
var resources embed.FS
func (dis Dis) Stack() component.Installable {
return dis.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "dis.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": dis.Config.DefaultHost(),
"LETSENCRYPT_HOST": dis.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
"CONFIG_PATH": dis.Config.ConfigPath,
"DEPLOY_ROOT": dis.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
},
TouchFiles: []string{dis.ResolverFile},
CopyContextFiles: []string{core.Executable},
})
}
func (dis Dis) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: dis.Config.CurrentExecutable(),
}
}

6
internal/component/installable.go
View file

@ -13,9 +13,9 @@ import (
// TODO: Move this package into components
// Installable represents a Stack that can be automatically installed from a set of resources
// StackWithResources represents a Stack that can be automatically installed from a set of resources.
// See the [Install] method.
type Installable struct {
type StackWithResources struct {
Stack
// Installable enabled installing several resources from a (potentially embedded) filesystem.
@ -42,7 +42,7 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(

8
internal/component/instances/wisski_stack.go
View file

@ -12,8 +12,8 @@ import (
var barrelResources embed.FS
// Barrel returns a stack representing the running WissKI Instance
func (wisski WissKI) Barrel() component.Installable {
return component.Installable{
func (wisski WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},
@ -48,8 +48,8 @@ func (wisski WissKI) Barrel() component.Installable {
var reserveResources embed.FS
// Reserve returns a stack representing the reserve instance
func (wisski WissKI) Reserve() component.Installable {
return component.Installable{
func (wisski WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},

35
internal/component/sql/backup.go Normal file
View file

@ -0,0 +1,35 @@
package sql
import (
"errors"
"os"
"github.com/tkw1536/goprogram/stream"
)
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
func (*SQL) BackupName() string {
return "sql.sql"
}
// Backup makes a backup of all SQL databases into the path dest.
func (sql *SQL) Backup(io stream.IOStream, dest string) error {
// open the file, TODO: Outsource this to context
writer, err := os.Create(dest)
if err != nil {
return err
}
defer writer.Close()
// run sqldump
io = io.Streams(writer, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}
if code != 0 {
return errSQLBackup
}
return nil
}

20
internal/component/sql/database.go
View file

@ -59,10 +59,8 @@ func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
return table, nil
}
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
// Backup makes a backup of the sql database into dest.
func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
// Snapshot makes a backup of the sql database into dest.
func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error {
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
@ -75,20 +73,6 @@ func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error
return nil
}
// BackupAll makes a backup of all sql databases
func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}
if code != 0 {
return errSQLBackup
}
return nil
}
// OpenShell executes a mysql shell command
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)

8
internal/component/sql/sql.go
View file

@ -22,13 +22,13 @@ func (SQL) Name() string {
return "sql"
}
//go:embed all:stack
//go:embed all:sql
var resources embed.FS
func (ssh SQL) Stack() component.Installable {
return ssh.ComponentBase.MakeStack(component.Installable{
func (ssh SQL) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
Resources: resources,
ContextPath: "stack",
ContextPath: "sql",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{

0
internal/component/sql/stack/docker-compose.yml → internal/component/sql/sql/docker-compose.yml
View file

4
internal/component/ssh/ssh.go
View file

@ -17,8 +17,8 @@ func (SSH) Name() string {
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() component.Installable {
return ssh.ComponentBase.MakeStack(component.Installable{
func (ssh SSH) Stack() component.StackWithResources {
return ssh.ComponentBase.MakeStack(component.StackWithResources{
Resources: resources,
ContextPath: "stack",
})

58
internal/component/triplestore/backup.go Normal file
View file

@ -0,0 +1,58 @@
package triplestore
import (
"encoding/json"
"io/fs"
"os"
"path/filepath"
"github.com/tkw1536/goprogram/stream"
)
func (ts *Triplestore) BackupName() string { return "triplestore" }
// Backup makes a backup of all Triplestore repositories databases into the path dest.
func (ts *Triplestore) Backup(io stream.IOStream, dest string) error {
// list all the repositories
repos, err := ts.listRepositories()
if err != nil {
return err
}
// create the base directory, todo: outsource this
if err := os.Mkdir(dest, fs.ModeDir); err != nil {
return err
}
// iterate over all the repositories
for _, repo := range repos {
if rErr := (func(repo Repository) error {
name := filepath.Join(dest, repo.ID+".nq")
// todo: outsource this
dest, err := os.Create(name)
if err != nil {
return err
}
defer dest.Close()
_, err = ts.Snapshot(dest, repo.ID)
return err
}(repo)); err == nil && rErr != nil {
err = rErr
}
}
return err
}
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
if err != nil {
return nil, err
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(&repos)
return
}

51
internal/component/triplestore/database.go
View file

@ -5,11 +5,8 @@ import (
"encoding/json"
"fmt"
"io"
"io/fs"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
@ -121,8 +118,8 @@ func (ts Triplestore) PurgeRepo(repo string) error {
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
// TriplestoreBackup backs up the repository named repo into the writer dst.
func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
// Snapshot snapshots the provided repository into dst
func (ts Triplestore) Snapshot(dst io.Writer, repo string) (int64, error) {
res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
if err != nil {
return 0, err
@ -146,50 +143,6 @@ type Repository struct {
Local bool `json:"local"`
}
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
if err != nil {
return nil, err
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(&repos)
return
}
// TriplestoreBackup backs up every graphdb instance into dst
func (ts Triplestore) BackupAll(dst string) error {
// list all the repositories
repos, err := ts.listRepositories()
if err != nil {
return err
}
// create the base directory
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
return err
}
// iterate over all the repositories
for _, repo := range repos {
if rErr := (func(repo Repository) error {
name := filepath.Join(dst, repo.ID+".nq")
dest, err := os.Create(name)
if err != nil {
return err
}
defer dest.Close()
_, err = ts.Backup(dest, repo.ID)
return err
}(repo)); err == nil && rErr != nil {
err = rErr
}
}
return err
}
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (ts Triplestore) Bootstrap(io stream.IOStream) error {

4
internal/component/triplestore/triplestore.go
View file

@ -26,8 +26,8 @@ func (Triplestore) Name() string {
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() component.Installable {
return ts.ComponentBase.MakeStack(component.Installable{
func (ts Triplestore) Stack() component.StackWithResources {
return ts.ComponentBase.MakeStack(component.StackWithResources{
Resources: resources,
ContextPath: "stack",

10
internal/component/web/web.go
View file

@ -17,7 +17,7 @@ func (Web) Name() string {
return "web"
}
func (web Web) Stack() component.Installable {
func (web Web) Stack() component.StackWithResources {
if web.Config.HTTPSEnabled() {
return web.stackHTTPS()
} else {
@ -29,8 +29,8 @@ func (web Web) Stack() component.Installable {
//go:embed web-https.env
var httpsResources embed.FS
func (web Web) stackHTTPS() component.Installable {
return web.MakeStack(component.Installable{
func (web Web) stackHTTPS() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
Resources: httpsResources,
ContextPath: "web-https",
EnvPath: "web-https.env",
@ -45,8 +45,8 @@ func (web Web) stackHTTPS() component.Installable {
//go:embed web-http.env
var httpResources embed.FS
func (web Web) stackHTTP() component.Installable {
return web.MakeStack(component.Installable{
func (web Web) stackHTTP() component.StackWithResources {
return web.MakeStack(component.StackWithResources{
Resources: httpResources,
ContextPath: "web-http",
EnvPath: "web-http.env",

122
internal/wisski/backup.go
View file

@ -11,11 +11,9 @@ import (
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
)
@ -36,13 +34,14 @@ type Backup struct {
// various error states, which are ignored when creating the snapshot
ErrPanic interface{}
// errors for the various components
ComponentErrors map[string]error
// SQL and triplestore errors
SQLErr error
TSErr error
TSErr error
// TODO: Make this proper
ConfigFileErr error
ConfigFilesManifest map[string]error
ConfigFileErr error
// Snapshots containing instances
InstanceListErr error
@ -76,17 +75,13 @@ func (backup Backup) Report(w io.Writer) (int, error) {
io.WriteString(cw, "\n")
io.WriteString(cw, "======= Errors =======\n")
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
fmt.Fprintf(cw, "SQLErr: %s\n", backup.SQLErr)
fmt.Fprintf(cw, "TSErr: %s\n", backup.TSErr)
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
fmt.Fprintf(cw, "Panic: %v\n", backup.ErrPanic)
fmt.Fprintf(cw, "Component Errors: %v\n", backup.ComponentErrors)
fmt.Fprintf(cw, "ConfigFileErr: %s\n", backup.ConfigFileErr)
fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
io.WriteString(cw, "\n")
io.WriteString(cw, "======= Config Files =======\n")
encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
io.WriteString(cw, "======= Snapshots =======\n")
for _, s := range backup.InstanceSnapshots {
io.WriteString(cw, s.String())
@ -103,6 +98,8 @@ func (backup Backup) Report(w io.Writer) (int, error) {
return cw.Sum()
}
// Backup makes a makes of the entire distillery.
// To make a backup, all [BackupComponents] will be invoked.
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
backup.Description = description
@ -123,75 +120,36 @@ func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription)
return
}
var errBackupSkipFile = errors.New("<file not found>")
type backupResult struct {
name string
err error
}
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
// create a wait group, and message channel
wg := &sync.WaitGroup{}
files := make(chan string, 4)
// backup the sql
wg.Add(1)
go func() {
defer wg.Done()
backups := dis.Backupable()
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
files <- sqlPath
files := make(chan string, len(backups)) // channel for files being added into the backups
results := make(chan backupResult, len(backups)) // channel for results to be stored into
backup.ComponentErrors = make(map[string]error, len(backups))
sql, err := os.Create(sqlPath)
if err != nil {
backup.SQLErr = err
return
}
defer sql.Close()
wg := &sync.WaitGroup{} // to wait for the results
wg.Add(len(backups))
for _, bc := range backups {
go func(bc component.Backupable) {
defer wg.Done()
// directly store the result
backup.SQLErr = dis.SQL().BackupAll(io, sql)
}()
// find the backup destination
dest := filepath.Join(backup.Description.Dest, bc.BackupName())
files <- dest
// backup the triplestore
wg.Add(1)
go func() {
defer wg.Done()
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
files <- tsPath
// directly store the result
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
}()
// backup configuration files
wg.Add(1)
go func() {
defer wg.Done()
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
backup.ConfigFileErr = err
return
}
configs := []string{
dis.Config.ConfigPath,
filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
dis.Config.SelfOverridesFile,
dis.Config.GlobalAuthorizedKeysFile,
}
backup.ConfigFilesManifest = make(map[string]error, len(configs))
for _, src := range configs {
if !fsx.IsFile(src) {
backup.ConfigFilesManifest[src] = errBackupSkipFile
continue
// make the backup and send the result!
results <- backupResult{
name: bc.Name(),
err: bc.Backup(io, dest),
}
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
// copy the config file and store the error message
files <- src
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
}
}()
}(bc)
}
// backup instances
wg.Add(1)
@ -230,10 +188,18 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
}()
// wait for the group, then close the message channel.
// finish processing all the results as soon as the group is done.
go func() {
defer close(results)
wg.Wait()
close(files)
}()
// finish the message processing once results are finished.
go func() {
defer close(files) // no more file processing!
for result := range results {
backup.ComponentErrors[result.name] = result.err
}
}()
for file := range files {

47
internal/wisski/component.go
View file

@ -6,7 +6,7 @@ import (
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
"github.com/FAU-CDI/wisski-distillery/internal/component/control"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
@ -23,11 +23,11 @@ import (
type components struct {
// installable components
web lazy.Lazy[*web.Web]
dis lazy.Lazy[*dis.Dis]
ssh lazy.Lazy[*ssh.SSH]
ts lazy.Lazy[*triplestore.Triplestore]
sql lazy.Lazy[*sql.SQL]
web lazy.Lazy[*web.Web]
control lazy.Lazy[*control.Control]
ssh lazy.Lazy[*ssh.SSH]
ts lazy.Lazy[*triplestore.Triplestore]
sql lazy.Lazy[*sql.SQL]
// other components
instances lazy.Lazy[*instances.Instances]
@ -67,23 +67,48 @@ func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C],
})
}
// Components returns all components that have a stack function
func (dis *Distillery) Components() []component.InstallableComponent {
return []component.InstallableComponent{
func (dis *Distillery) ComponentsX() []component.Component {
return []component.Component{
dis.Web(),
dis.Dis(),
dis.SSH(),
dis.Triplestore(),
dis.SQL(),
dis.Instances(),
}
}
// Backupable returns all the components that can be backuped up.
func (dis *Distillery) Backupable() []component.Backupable {
return getComponents[component.Backupable](dis)
}
// Installables returns all components that can be installed
func (dis *Distillery) Installables() []component.Installable {
return getComponents[component.Installable](dis)
}
func getComponents[C component.Component](dis *Distillery) (result []C) {
all := dis.ComponentsX()
result = make([]C, 0, len(all))
for _, c := range all {
sc, ok := c.(C)
if !ok {
continue
}
result = append(result, sc)
}
return
}
func (dis *Distillery) Web() *web.Web {
return makeComponent(dis, &dis.components.web, nil)
}
func (d *Distillery) Dis() *dis.Dis {
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
func (d *Distillery) Dis() *control.Control {
return makeComponent(d, &d.components.control, func(ddis *control.Control) {
ddis.ResolverFile = core.PrefixConfig
ddis.Instances = d.Instances()
})

4
internal/wisski/snapshot.go
View file

@ -244,7 +244,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
defer nquads.Close()
// directly store the result
_, err = dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
_, err = dis.Triplestore().Snapshot(nquads, instance.GraphDBRepository)
return err
}, &snapshot.ErrTriplestore)
@ -260,7 +260,7 @@ func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, inst
defer sql.Close()
// directly store the result
return dis.SQL().Backup(io, sql, instance.SqlDatabase)
return dis.SQL().Snapshot(io, sql, instance.SqlDatabase)
}, &snapshot.ErrSQL)
// wait for the group!