Make 'system_update' more generic

cmd/system_update.go

@@ -3,12 +3,10 @@ package cmd
 import (
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
-	"github.com/FAU-CDI/wisski-distillery/internal/config"
 	"github.com/FAU-CDI/wisski-distillery/internal/core"
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
-	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
 	"github.com/tkw1536/goprogram/exit"
 	"github.com/tkw1536/goprogram/parser"
 )
@@ -54,18 +52,8 @@ var errBoostrapFailedToCreateDirectory = exit.Error{
 	ExitCode: exit.ExitGeneric,
 }
 
-var errBootstrapFailedRuntime = exit.Error{
+var errBootstrapComponent = exit.Error{
-	Message:  "failed to update runtime: %s",
+	Message:  "Unable to bootstrap %s: %s",
-	ExitCode: exit.ExitGeneric,
-}
-
-var errBootstrapTriplestore = exit.Error{
-	Message:  "Unable to bootstrap Triplestore: %s",
-	ExitCode: exit.ExitGeneric,
-}
-
-var errBootstrapSQL = exit.Error{
-	Message:  "Unable to bootstrap SQL: %s",
 	ExitCode: exit.ExitGeneric,
 }
 
@@ -119,45 +107,40 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
 
 	if err := logging.LogOperation(func() error {
 		for _, component := range dis.Installables() {
+			name := component.Name()
 			stack := component.Stack(dis.Core.Environment)
 			ctx := component.Context(ctx)
+
 			if err := logging.LogOperation(func() error {
 				return stack.Install(dis.Core.Environment, context.IOStream, ctx)
-			}, context.IOStream, "Installing docker stack %q", component.Name()); err != nil {
+			}, context.IOStream, "Installing Docker Stack %q", name); err != nil {
 				return err
 			}
 
 			if err := logging.LogOperation(func() error {
 				return stack.Update(context.IOStream, true)
-			}, context.IOStream, "Updating docker stack %q", component.Name()); err != nil {
+			}, context.IOStream, "Updating Docker Stack: %q", name); err != nil {
 				return err
 			}
 		}
 		return nil
-	}, context.IOStream, "Updating Components"); err != nil {
+	}, context.IOStream, "Performing Stack Updates"); err != nil {
 		return err
 	}
 
 	if err := logging.LogOperation(func() error {
-		return unpack.InstallDir(dis.Core.Environment, dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) {
+		for _, component := range dis.Updateable() {
-			context.Printf("[copy]  %s\n", dst)
+			name := component.Name()
-		})
+			if err := logging.LogOperation(func() error {
-	}, context.IOStream, "Unpacking Runtime Components"); err != nil {
+				return component.Update(context.IOStream)
-		return errBootstrapFailedRuntime.WithMessageF(err)
+			}, context.IOStream, "Updating Component: %s", name); err != nil {
+				return errBootstrapComponent.WithMessageF(name, err)
+			}
+		}
+		return nil
+	}, context.IOStream, "Performing Component Updates"); err != nil {
+		return err
 	}
-
-	if err := logging.LogOperation(func() error {
-		return dis.SQL().Bootstrap(context.IOStream)
-	}, context.IOStream, "Bootstraping SQL database"); err != nil {
-		return errBootstrapSQL.WithMessageF(err)
-	}
-
-	if err := logging.LogOperation(func() error {
-		return dis.Triplestore().Bootstrap(context.IOStream)
-	}, context.IOStream, "Bootstraping Triplestore"); err != nil {
-		return errBootstrapTriplestore.WithMessageF(err)
-	}
-
 	// TODO: Register cronjob in /etc/cron.d!
 
 	logging.LogMessage(context.IOStream, "System has been updated")

internal/component/component.go

@@ -32,20 +32,6 @@ type Component interface {
 	Base() *ComponentBase
 }
 
-// Installable implements an installable component.
-type Installable interface {
-	Component
-
-	// Stack can be used to gain access to the "docker compose" stack.
-	//
-	// This should internally call [ComponentBase.MakeStack]
-	Stack(env environment.Environment) StackWithResources
-
-	// Context returns a new InstallationContext to be used during installation from the command line.
-	// Typically this should just pass through the parent, but might perform other tasks.
-	Context(parent InstallationContext) InstallationContext
-}
-
 // ComponentBase implements base functionality for a component
 type ComponentBase struct {
 	Core        // the core of the associated distillery

internal/component/installable.go

@@ -1,120 +1,33 @@
 package component
 
 import (
-	"io/fs"
-	"path/filepath"
-
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
-	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
-	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
-	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 )
 
-// TODO: Move this package into components
+// Installable implements an installable component.
+type Installable interface {
+	Component
 
-// StackWithResources represents a Stack that can be automatically installed from a set of resources.
+	// Stack can be used to gain access to the "docker compose" stack.
-// See the [Install] method.
-type StackWithResources struct {
-	Stack
-
-	// Installable enabled installing several resources from a (potentially embedded) filesystem.
 	//
-	// The Resources holds these, with appropriate resources specified below.
+	// This should internally call [ComponentBase.MakeStack]
-	// These all refer to paths within the Resource filesystem.
+	Stack(env environment.Environment) StackWithResources
-	Resources   fs.FS
-	ContextPath string            // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
-	EnvPath     string            // the '.env' template, will be installed using [unpack.InstallTemplate].
-	EnvContext  map[string]string // context when instantiating the '.env' template
 
-	CopyContextFiles []string // Files to copy from the installation context
+	// Context returns a new InstallationContext to be used during installation from the command line.
-
+	// Typically this should just pass through the parent, but might perform other tasks.
-	MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
+	Context(parent InstallationContext) InstallationContext
-	MakeDirs     []string    // directories to ensure that exist
-
-	TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
 }
 
-// InstallationContext is a context to install data in
+// Updatable represents a component with an Update method.
-type InstallationContext map[string]string
+type Updatable interface {
+	Component
 
-// Install installs or updates this stack into the directory specified by stack.Stack().
+	// Update updates or initializes the provided components.
-//
+	// It is called after the component has been installed (if applicable).
-// Installation is non-interactive, but will provide debugging output onto io.
+	//
-// InstallationContext
+	// It may send output to the provided stream.
-func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
+	//
-	if is.ContextPath != "" {
+	// Updating should be idempotent, meaning running it multiple times must not break the existing system.
-		// setup the base files
+	Update(stream stream.IOStream) error
-		if err := unpack.InstallDir(
-			env,
-			is.Dir,
-			is.ContextPath,
-			is.Resources,
-			func(dst, src string) {
-				io.Printf("[install] %s\n", dst)
-			},
-		); err != nil {
-			return err
-		}
-	}
-
-	// configure .env
-	envDest := filepath.Join(is.Dir, ".env")
-	if is.EnvPath != "" && is.EnvContext != nil {
-		io.Printf("[config]  %s\n", envDest)
-		if err := unpack.InstallTemplate(
-			env,
-			envDest,
-			is.EnvContext,
-			is.EnvPath,
-			is.Resources,
-		); err != nil {
-			return err
-		}
-	}
-
-	// make sure that certain dirs exist
-	for _, name := range is.MakeDirs {
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		io.Printf("[make]    %s\n", dst)
-		if is.MakeDirsPerm == fs.FileMode(0) {
-			is.MakeDirsPerm = environment.DefaultDirPerm
-		}
-		if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
-			return err
-		}
-	}
-
-	// copy files from the context!
-	for _, name := range is.CopyContextFiles {
-		// find the source!
-		src, ok := context[name]
-		if !ok {
-			return errors.Errorf("Missing file from context: %q", src)
-		}
-
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		// copy over file from context
-		io.Printf("[copy]    %s (from %s)\n", dst, src)
-		if err := fsx.CopyFile(env, dst, src); err != nil {
-			return errors.Wrapf(err, "Unable to copy file %s", src)
-		}
-	}
-
-	// make sure that certain files exist
-	for _, name := range is.TouchFiles {
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		io.Printf("[touch]   %s\n", dst)
-		if err := fsx.Touch(env, dst); err != nil {
-			return err
-		}
-	}
-
-	return nil
 }

internal/component/instances/runtime.go

@@ -0,0 +1,29 @@
+package instances
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var errBootstrapFailedRuntime = exit.Error{
+	Message:  "failed to update runtime",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Runtime contains runtime resources to be installed into any instance
+//go:embed all:runtime
+var runtimeResources embed.FS
+
+// Update installs or updates runtime components needed by this component.
+func (instances *Instances) Update(stream stream.IOStream) error {
+	err := unpack.InstallDir(instances.Core.Environment, instances.Config.RuntimeDir(), "runtime", runtimeResources, func(dst, src string) {
+		stream.Printf("[copy]  %s\n", dst)
+	})
+	if err != nil {
+		return errBootstrapFailedRuntime.Wrap(err)
+	}
+	return nil
+}

internal/component/sql/update.go

@@ -14,8 +14,8 @@ var errSQLUnableToCreateUser = errors.New("unable to create administrative user"
 var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name")
 var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
 
-// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
+// Update initializes or updates the SQL database.
-func (sql *SQL) Bootstrap(io stream.IOStream) error {
+func (sql *SQL) Update(io stream.IOStream) error {
 	if err := sql.WaitShell(); err != nil {
 		return err
 	}

internal/component/stack.go

@@ -4,9 +4,13 @@ package component
 import (
 	"bufio"
 	"bytes"
-	"errors"
+	"io/fs"
+	"path/filepath"
 
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
+	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 )
 
@@ -183,3 +187,109 @@ func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
 	}
 	return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
 }
+
+// StackWithResources represents a Stack that can be automatically installed from a set of resources.
+// See the [Install] method.
+type StackWithResources struct {
+	Stack
+
+	// Installable enabled installing several resources from a (potentially embedded) filesystem.
+	//
+	// The Resources holds these, with appropriate resources specified below.
+	// These all refer to paths within the Resource filesystem.
+	Resources   fs.FS
+	ContextPath string            // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
+	EnvPath     string            // the '.env' template, will be installed using [unpack.InstallTemplate].
+	EnvContext  map[string]string // context when instantiating the '.env' template
+
+	CopyContextFiles []string // Files to copy from the installation context
+
+	MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
+	MakeDirs     []string    // directories to ensure that exist
+
+	TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
+}
+
+// InstallationContext is a context to install data in
+type InstallationContext map[string]string
+
+// Install installs or updates this stack into the directory specified by stack.Stack().
+//
+// Installation is non-interactive, but will provide debugging output onto io.
+// InstallationContext
+func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
+	if is.ContextPath != "" {
+		// setup the base files
+		if err := unpack.InstallDir(
+			env,
+			is.Dir,
+			is.ContextPath,
+			is.Resources,
+			func(dst, src string) {
+				io.Printf("[install] %s\n", dst)
+			},
+		); err != nil {
+			return err
+		}
+	}
+
+	// configure .env
+	envDest := filepath.Join(is.Dir, ".env")
+	if is.EnvPath != "" && is.EnvContext != nil {
+		io.Printf("[config]  %s\n", envDest)
+		if err := unpack.InstallTemplate(
+			env,
+			envDest,
+			is.EnvContext,
+			is.EnvPath,
+			is.Resources,
+		); err != nil {
+			return err
+		}
+	}
+
+	// make sure that certain dirs exist
+	for _, name := range is.MakeDirs {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[make]    %s\n", dst)
+		if is.MakeDirsPerm == fs.FileMode(0) {
+			is.MakeDirsPerm = environment.DefaultDirPerm
+		}
+		if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
+			return err
+		}
+	}
+
+	// copy files from the context!
+	for _, name := range is.CopyContextFiles {
+		// find the source!
+		src, ok := context[name]
+		if !ok {
+			return errors.Errorf("Missing file from context: %q", src)
+		}
+
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		// copy over file from context
+		io.Printf("[copy]    %s (from %s)\n", dst, src)
+		if err := fsx.CopyFile(env, dst, src); err != nil {
+			return errors.Wrapf(err, "Unable to copy file %s", src)
+		}
+	}
+
+	// make sure that certain files exist
+	for _, name := range is.TouchFiles {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[touch]   %s\n", dst)
+		if err := fsx.Touch(env, dst); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/component/triplestore/database.go

@@ -3,15 +3,12 @@ package triplestore
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
 
-	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/wait"
 	"github.com/pkg/errors"
-	"github.com/tkw1536/goprogram/stream"
 )
 
 type TriplestoreUserPayload struct {
@@ -148,59 +145,3 @@ type Repository struct {
 	Writable   bool   `json:"writable"`
 	Local      bool   `json:"local"`
 }
-
-var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
-
-func (ts Triplestore) Bootstrap(io stream.IOStream) error {
-	logging.LogMessage(io, "Waiting for Triplestore")
-	if err := ts.Wait(); err != nil {
-		return err
-	}
-
-	logging.LogMessage(io, "Resetting admin user password")
-	{
-		res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
-			Password: ts.Config.TriplestoreAdminPassword,
-			AppSettings: TriplestoreUserAppSettings{
-				DefaultInference:      true,
-				DefaultVisGraphSchema: true,
-				DefaultSameas:         true,
-				IgnoreSharedQueries:   false,
-				ExecuteCount:          true,
-			},
-			GrantedAuthorities: []string{"ROLE_ADMIN"},
-		}, "", "")
-		if err != nil {
-			return fmt.Errorf("failed to create triplestore user: %s", err)
-		}
-		defer res.Body.Close()
-
-		switch res.StatusCode {
-		case http.StatusOK:
-			// we set the password => requests are unauthorized
-			// so we still need to enable security (see below!)
-		case http.StatusUnauthorized:
-			// a password is needed => security is already enabled.
-			// the password may or may not work, but that's a problem for later
-			logging.LogMessage(io, "Security is already enabled")
-			return nil
-		default:
-			return fmt.Errorf("failed to create triplestore user: %s", err)
-		}
-	}
-
-	logging.LogMessage(io, "Enabling Triplestore security")
-	{
-		res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
-		if err != nil {
-			return fmt.Errorf("failed to enable triplestore security: %s", err)
-		}
-		defer res.Body.Close()
-
-		if res.StatusCode != http.StatusOK {
-			return errTriplestoreFailedSecurity
-		}
-
-		return nil
-	}
-}

internal/component/triplestore/update.go

@@ -0,0 +1,66 @@
+package triplestore
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
+
+func (ts Triplestore) Update(io stream.IOStream) error {
+	logging.LogMessage(io, "Waiting for Triplestore")
+	if err := ts.Wait(); err != nil {
+		return err
+	}
+
+	logging.LogMessage(io, "Resetting admin user password")
+	{
+		res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
+			Password: ts.Config.TriplestoreAdminPassword,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{"ROLE_ADMIN"},
+		}, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+		defer res.Body.Close()
+
+		switch res.StatusCode {
+		case http.StatusOK:
+			// we set the password => requests are unauthorized
+			// so we still need to enable security (see below!)
+		case http.StatusUnauthorized:
+			// a password is needed => security is already enabled.
+			// the password may or may not work, but that's a problem for later
+			logging.LogMessage(io, "Security is already enabled")
+			return nil
+		default:
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+	}
+
+	logging.LogMessage(io, "Enabling Triplestore security")
+	{
+		res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to enable triplestore security: %s", err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return errTriplestoreFailedSecurity
+		}
+
+		return nil
+	}
+}

internal/config/runtime.go

@@ -1,14 +1,9 @@
 package config
 
 import (
-	"embed"
 	"path/filepath"
 )
 
-// Runtime contains runtime resources to be installed into any instance
-//go:embed all:runtime
-var Runtime embed.FS
-
 // RuntimeDir returns the path to the runtime directory
 func (cfg Config) RuntimeDir() string {
 	return filepath.Join(cfg.DeployRoot, "runtime")

internal/wisski/component.go

@@ -88,6 +88,11 @@ func (dis *Distillery) Installables() []component.Installable {
 	return getComponents[component.Installable](dis)
 }
 
+// Installables returns all components that can be installed
+func (dis *Distillery) Updateable() []component.Updatable {
+	return getComponents[component.Updatable](dis)
+}
+
 func getComponents[C component.Component](dis *Distillery) (result []C) {
 	all := dis.ComponentsX()