From 881b538dff7d1611b82ee94067866f07c45b40df Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Mon, 19 Sep 2022 14:56:28 +0200 Subject: [PATCH] Make 'system_update' more generic --- cmd/system_update.go | 53 +++----- internal/component/component.go | 14 -- internal/component/installable.go | 125 +++--------------- internal/component/instances/runtime.go | 29 ++++ .../instances}/runtime/README | 0 .../instances}/runtime/blind_update.sh | 0 .../instances}/runtime/create_admin.sh | 0 .../instances}/runtime/cron.sh | 0 .../instances}/runtime/install_colorbox.sh | 0 .../instances}/runtime/patch_easyrdf.sh | 0 .../instances}/runtime/patch_triples.sh | 0 .../instances}/runtime/use_wisski.sh | 0 .../instances}/runtime/wisski_2x_3x.sh | 0 .../component/sql/{bootstrap.go => update.go} | 4 +- internal/component/stack.go | 112 +++++++++++++++- internal/component/triplestore/database.go | 59 --------- internal/component/triplestore/update.go | 66 +++++++++ internal/config/runtime.go | 5 - internal/wisski/component.go | 5 + 19 files changed, 250 insertions(+), 222 deletions(-) create mode 100644 internal/component/instances/runtime.go rename internal/{config => component/instances}/runtime/README (100%) rename internal/{config => component/instances}/runtime/blind_update.sh (100%) rename internal/{config => component/instances}/runtime/create_admin.sh (100%) rename internal/{config => component/instances}/runtime/cron.sh (100%) rename internal/{config => component/instances}/runtime/install_colorbox.sh (100%) rename internal/{config => component/instances}/runtime/patch_easyrdf.sh (100%) rename internal/{config => component/instances}/runtime/patch_triples.sh (100%) rename internal/{config => component/instances}/runtime/use_wisski.sh (100%) rename internal/{config => component/instances}/runtime/wisski_2x_3x.sh (100%) rename internal/component/sql/{bootstrap.go => update.go} (92%) create mode 100644 internal/component/triplestore/update.go diff --git a/cmd/system_update.go b/cmd/system_update.go index c46c7e7..84c837e 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -3,12 +3,10 @@ package cmd import ( wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/component" - "github.com/FAU-CDI/wisski-distillery/internal/config" "github.com/FAU-CDI/wisski-distillery/internal/core" "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" "github.com/FAU-CDI/wisski-distillery/pkg/logging" - "github.com/FAU-CDI/wisski-distillery/pkg/unpack" "github.com/tkw1536/goprogram/exit" "github.com/tkw1536/goprogram/parser" ) @@ -54,18 +52,8 @@ var errBoostrapFailedToCreateDirectory = exit.Error{ ExitCode: exit.ExitGeneric, } -var errBootstrapFailedRuntime = exit.Error{ - Message: "failed to update runtime: %s", - ExitCode: exit.ExitGeneric, -} - -var errBootstrapTriplestore = exit.Error{ - Message: "Unable to bootstrap Triplestore: %s", - ExitCode: exit.ExitGeneric, -} - -var errBootstrapSQL = exit.Error{ - Message: "Unable to bootstrap SQL: %s", +var errBootstrapComponent = exit.Error{ + Message: "Unable to bootstrap %s: %s", ExitCode: exit.ExitGeneric, } @@ -119,45 +107,40 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { if err := logging.LogOperation(func() error { for _, component := range dis.Installables() { + name := component.Name() stack := component.Stack(dis.Core.Environment) ctx := component.Context(ctx) + if err := logging.LogOperation(func() error { return stack.Install(dis.Core.Environment, context.IOStream, ctx) - }, context.IOStream, "Installing docker stack %q", component.Name()); err != nil { + }, context.IOStream, "Installing Docker Stack %q", name); err != nil { return err } if err := logging.LogOperation(func() error { return stack.Update(context.IOStream, true) - }, context.IOStream, "Updating docker stack %q", component.Name()); err != nil { + }, context.IOStream, "Updating Docker Stack: %q", name); err != nil { return err } } return nil - }, context.IOStream, "Updating Components"); err != nil { + }, context.IOStream, "Performing Stack Updates"); err != nil { return err } if err := logging.LogOperation(func() error { - return unpack.InstallDir(dis.Core.Environment, dis.Config.RuntimeDir(), "runtime", config.Runtime, func(dst, src string) { - context.Printf("[copy] %s\n", dst) - }) - }, context.IOStream, "Unpacking Runtime Components"); err != nil { - return errBootstrapFailedRuntime.WithMessageF(err) + for _, component := range dis.Updateable() { + name := component.Name() + if err := logging.LogOperation(func() error { + return component.Update(context.IOStream) + }, context.IOStream, "Updating Component: %s", name); err != nil { + return errBootstrapComponent.WithMessageF(name, err) + } + } + return nil + }, context.IOStream, "Performing Component Updates"); err != nil { + return err } - - if err := logging.LogOperation(func() error { - return dis.SQL().Bootstrap(context.IOStream) - }, context.IOStream, "Bootstraping SQL database"); err != nil { - return errBootstrapSQL.WithMessageF(err) - } - - if err := logging.LogOperation(func() error { - return dis.Triplestore().Bootstrap(context.IOStream) - }, context.IOStream, "Bootstraping Triplestore"); err != nil { - return errBootstrapTriplestore.WithMessageF(err) - } - // TODO: Register cronjob in /etc/cron.d! logging.LogMessage(context.IOStream, "System has been updated") diff --git a/internal/component/component.go b/internal/component/component.go index 0da5a49..314fe64 100644 --- a/internal/component/component.go +++ b/internal/component/component.go @@ -32,20 +32,6 @@ type Component interface { Base() *ComponentBase } -// Installable implements an installable component. -type Installable interface { - Component - - // Stack can be used to gain access to the "docker compose" stack. - // - // This should internally call [ComponentBase.MakeStack] - Stack(env environment.Environment) StackWithResources - - // Context returns a new InstallationContext to be used during installation from the command line. - // Typically this should just pass through the parent, but might perform other tasks. - Context(parent InstallationContext) InstallationContext -} - // ComponentBase implements base functionality for a component type ComponentBase struct { Core // the core of the associated distillery diff --git a/internal/component/installable.go b/internal/component/installable.go index 4cea911..2e080dd 100644 --- a/internal/component/installable.go +++ b/internal/component/installable.go @@ -1,120 +1,33 @@ package component import ( - "io/fs" - "path/filepath" - "github.com/FAU-CDI/wisski-distillery/pkg/environment" - "github.com/FAU-CDI/wisski-distillery/pkg/fsx" - "github.com/FAU-CDI/wisski-distillery/pkg/unpack" - "github.com/pkg/errors" "github.com/tkw1536/goprogram/stream" ) -// TODO: Move this package into components +// Installable implements an installable component. +type Installable interface { + Component -// StackWithResources represents a Stack that can be automatically installed from a set of resources. -// See the [Install] method. -type StackWithResources struct { - Stack - - // Installable enabled installing several resources from a (potentially embedded) filesystem. + // Stack can be used to gain access to the "docker compose" stack. // - // The Resources holds these, with appropriate resources specified below. - // These all refer to paths within the Resource filesystem. - Resources fs.FS - ContextPath string // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'. - EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate]. - EnvContext map[string]string // context when instantiating the '.env' template + // This should internally call [ComponentBase.MakeStack] + Stack(env environment.Environment) StackWithResources - CopyContextFiles []string // Files to copy from the installation context - - MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate] - MakeDirs []string // directories to ensure that exist - - TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs + // Context returns a new InstallationContext to be used during installation from the command line. + // Typically this should just pass through the parent, but might perform other tasks. + Context(parent InstallationContext) InstallationContext } -// InstallationContext is a context to install data in -type InstallationContext map[string]string +// Updatable represents a component with an Update method. +type Updatable interface { + Component -// Install installs or updates this stack into the directory specified by stack.Stack(). -// -// Installation is non-interactive, but will provide debugging output onto io. -// InstallationContext -func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error { - if is.ContextPath != "" { - // setup the base files - if err := unpack.InstallDir( - env, - is.Dir, - is.ContextPath, - is.Resources, - func(dst, src string) { - io.Printf("[install] %s\n", dst) - }, - ); err != nil { - return err - } - } - - // configure .env - envDest := filepath.Join(is.Dir, ".env") - if is.EnvPath != "" && is.EnvContext != nil { - io.Printf("[config] %s\n", envDest) - if err := unpack.InstallTemplate( - env, - envDest, - is.EnvContext, - is.EnvPath, - is.Resources, - ); err != nil { - return err - } - } - - // make sure that certain dirs exist - for _, name := range is.MakeDirs { - // find the destination! - dst := filepath.Join(is.Dir, name) - - io.Printf("[make] %s\n", dst) - if is.MakeDirsPerm == fs.FileMode(0) { - is.MakeDirsPerm = environment.DefaultDirPerm - } - if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil { - return err - } - } - - // copy files from the context! - for _, name := range is.CopyContextFiles { - // find the source! - src, ok := context[name] - if !ok { - return errors.Errorf("Missing file from context: %q", src) - } - - // find the destination! - dst := filepath.Join(is.Dir, name) - - // copy over file from context - io.Printf("[copy] %s (from %s)\n", dst, src) - if err := fsx.CopyFile(env, dst, src); err != nil { - return errors.Wrapf(err, "Unable to copy file %s", src) - } - } - - // make sure that certain files exist - for _, name := range is.TouchFiles { - // find the destination! - dst := filepath.Join(is.Dir, name) - - io.Printf("[touch] %s\n", dst) - if err := fsx.Touch(env, dst); err != nil { - return err - } - } - - return nil + // Update updates or initializes the provided components. + // It is called after the component has been installed (if applicable). + // + // It may send output to the provided stream. + // + // Updating should be idempotent, meaning running it multiple times must not break the existing system. + Update(stream stream.IOStream) error } diff --git a/internal/component/instances/runtime.go b/internal/component/instances/runtime.go new file mode 100644 index 0000000..1f42dee --- /dev/null +++ b/internal/component/instances/runtime.go @@ -0,0 +1,29 @@ +package instances + +import ( + "embed" + + "github.com/FAU-CDI/wisski-distillery/pkg/unpack" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/stream" +) + +var errBootstrapFailedRuntime = exit.Error{ + Message: "failed to update runtime", + ExitCode: exit.ExitGeneric, +} + +// Runtime contains runtime resources to be installed into any instance +//go:embed all:runtime +var runtimeResources embed.FS + +// Update installs or updates runtime components needed by this component. +func (instances *Instances) Update(stream stream.IOStream) error { + err := unpack.InstallDir(instances.Core.Environment, instances.Config.RuntimeDir(), "runtime", runtimeResources, func(dst, src string) { + stream.Printf("[copy] %s\n", dst) + }) + if err != nil { + return errBootstrapFailedRuntime.Wrap(err) + } + return nil +} diff --git a/internal/config/runtime/README b/internal/component/instances/runtime/README similarity index 100% rename from internal/config/runtime/README rename to internal/component/instances/runtime/README diff --git a/internal/config/runtime/blind_update.sh b/internal/component/instances/runtime/blind_update.sh similarity index 100% rename from internal/config/runtime/blind_update.sh rename to internal/component/instances/runtime/blind_update.sh diff --git a/internal/config/runtime/create_admin.sh b/internal/component/instances/runtime/create_admin.sh similarity index 100% rename from internal/config/runtime/create_admin.sh rename to internal/component/instances/runtime/create_admin.sh diff --git a/internal/config/runtime/cron.sh b/internal/component/instances/runtime/cron.sh similarity index 100% rename from internal/config/runtime/cron.sh rename to internal/component/instances/runtime/cron.sh diff --git a/internal/config/runtime/install_colorbox.sh b/internal/component/instances/runtime/install_colorbox.sh similarity index 100% rename from internal/config/runtime/install_colorbox.sh rename to internal/component/instances/runtime/install_colorbox.sh diff --git a/internal/config/runtime/patch_easyrdf.sh b/internal/component/instances/runtime/patch_easyrdf.sh similarity index 100% rename from internal/config/runtime/patch_easyrdf.sh rename to internal/component/instances/runtime/patch_easyrdf.sh diff --git a/internal/config/runtime/patch_triples.sh b/internal/component/instances/runtime/patch_triples.sh similarity index 100% rename from internal/config/runtime/patch_triples.sh rename to internal/component/instances/runtime/patch_triples.sh diff --git a/internal/config/runtime/use_wisski.sh b/internal/component/instances/runtime/use_wisski.sh similarity index 100% rename from internal/config/runtime/use_wisski.sh rename to internal/component/instances/runtime/use_wisski.sh diff --git a/internal/config/runtime/wisski_2x_3x.sh b/internal/component/instances/runtime/wisski_2x_3x.sh similarity index 100% rename from internal/config/runtime/wisski_2x_3x.sh rename to internal/component/instances/runtime/wisski_2x_3x.sh diff --git a/internal/component/sql/bootstrap.go b/internal/component/sql/update.go similarity index 92% rename from internal/component/sql/bootstrap.go rename to internal/component/sql/update.go index ef18c6b..7e574ff 100644 --- a/internal/component/sql/bootstrap.go +++ b/internal/component/sql/update.go @@ -14,8 +14,8 @@ var errSQLUnableToCreateUser = errors.New("unable to create administrative user" var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name") var errSQLUnableToCreate = errors.New("unable to create bookkeeping database") -// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date -func (sql *SQL) Bootstrap(io stream.IOStream) error { +// Update initializes or updates the SQL database. +func (sql *SQL) Update(io stream.IOStream) error { if err := sql.WaitShell(); err != nil { return err } diff --git a/internal/component/stack.go b/internal/component/stack.go index a832f0a..809741b 100644 --- a/internal/component/stack.go +++ b/internal/component/stack.go @@ -4,9 +4,13 @@ package component import ( "bufio" "bytes" - "errors" + "io/fs" + "path/filepath" "github.com/FAU-CDI/wisski-distillery/pkg/environment" + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" + "github.com/FAU-CDI/wisski-distillery/pkg/unpack" + "github.com/pkg/errors" "github.com/tkw1536/goprogram/stream" ) @@ -183,3 +187,109 @@ func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) { } return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil } + +// StackWithResources represents a Stack that can be automatically installed from a set of resources. +// See the [Install] method. +type StackWithResources struct { + Stack + + // Installable enabled installing several resources from a (potentially embedded) filesystem. + // + // The Resources holds these, with appropriate resources specified below. + // These all refer to paths within the Resource filesystem. + Resources fs.FS + ContextPath string // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'. + EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate]. + EnvContext map[string]string // context when instantiating the '.env' template + + CopyContextFiles []string // Files to copy from the installation context + + MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate] + MakeDirs []string // directories to ensure that exist + + TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs +} + +// InstallationContext is a context to install data in +type InstallationContext map[string]string + +// Install installs or updates this stack into the directory specified by stack.Stack(). +// +// Installation is non-interactive, but will provide debugging output onto io. +// InstallationContext +func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error { + if is.ContextPath != "" { + // setup the base files + if err := unpack.InstallDir( + env, + is.Dir, + is.ContextPath, + is.Resources, + func(dst, src string) { + io.Printf("[install] %s\n", dst) + }, + ); err != nil { + return err + } + } + + // configure .env + envDest := filepath.Join(is.Dir, ".env") + if is.EnvPath != "" && is.EnvContext != nil { + io.Printf("[config] %s\n", envDest) + if err := unpack.InstallTemplate( + env, + envDest, + is.EnvContext, + is.EnvPath, + is.Resources, + ); err != nil { + return err + } + } + + // make sure that certain dirs exist + for _, name := range is.MakeDirs { + // find the destination! + dst := filepath.Join(is.Dir, name) + + io.Printf("[make] %s\n", dst) + if is.MakeDirsPerm == fs.FileMode(0) { + is.MakeDirsPerm = environment.DefaultDirPerm + } + if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil { + return err + } + } + + // copy files from the context! + for _, name := range is.CopyContextFiles { + // find the source! + src, ok := context[name] + if !ok { + return errors.Errorf("Missing file from context: %q", src) + } + + // find the destination! + dst := filepath.Join(is.Dir, name) + + // copy over file from context + io.Printf("[copy] %s (from %s)\n", dst, src) + if err := fsx.CopyFile(env, dst, src); err != nil { + return errors.Wrapf(err, "Unable to copy file %s", src) + } + } + + // make sure that certain files exist + for _, name := range is.TouchFiles { + // find the destination! + dst := filepath.Join(is.Dir, name) + + io.Printf("[touch] %s\n", dst) + if err := fsx.Touch(env, dst); err != nil { + return err + } + } + + return nil +} diff --git a/internal/component/triplestore/database.go b/internal/component/triplestore/database.go index f765bad..6fae113 100644 --- a/internal/component/triplestore/database.go +++ b/internal/component/triplestore/database.go @@ -3,15 +3,12 @@ package triplestore import ( "bytes" "encoding/json" - "fmt" "io" "mime/multipart" "net/http" - "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/wait" "github.com/pkg/errors" - "github.com/tkw1536/goprogram/stream" ) type TriplestoreUserPayload struct { @@ -148,59 +145,3 @@ type Repository struct { Writable bool `json:"writable"` Local bool `json:"local"` } - -var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK") - -func (ts Triplestore) Bootstrap(io stream.IOStream) error { - logging.LogMessage(io, "Waiting for Triplestore") - if err := ts.Wait(); err != nil { - return err - } - - logging.LogMessage(io, "Resetting admin user password") - { - res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{ - Password: ts.Config.TriplestoreAdminPassword, - AppSettings: TriplestoreUserAppSettings{ - DefaultInference: true, - DefaultVisGraphSchema: true, - DefaultSameas: true, - IgnoreSharedQueries: false, - ExecuteCount: true, - }, - GrantedAuthorities: []string{"ROLE_ADMIN"}, - }, "", "") - if err != nil { - return fmt.Errorf("failed to create triplestore user: %s", err) - } - defer res.Body.Close() - - switch res.StatusCode { - case http.StatusOK: - // we set the password => requests are unauthorized - // so we still need to enable security (see below!) - case http.StatusUnauthorized: - // a password is needed => security is already enabled. - // the password may or may not work, but that's a problem for later - logging.LogMessage(io, "Security is already enabled") - return nil - default: - return fmt.Errorf("failed to create triplestore user: %s", err) - } - } - - logging.LogMessage(io, "Enabling Triplestore security") - { - res, err := ts.OpenRaw("POST", "/rest/security", true, "", "") - if err != nil { - return fmt.Errorf("failed to enable triplestore security: %s", err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return errTriplestoreFailedSecurity - } - - return nil - } -} diff --git a/internal/component/triplestore/update.go b/internal/component/triplestore/update.go new file mode 100644 index 0000000..d6aefb9 --- /dev/null +++ b/internal/component/triplestore/update.go @@ -0,0 +1,66 @@ +package triplestore + +import ( + "fmt" + "net/http" + + "github.com/FAU-CDI/wisski-distillery/pkg/logging" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/stream" +) + +var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK") + +func (ts Triplestore) Update(io stream.IOStream) error { + logging.LogMessage(io, "Waiting for Triplestore") + if err := ts.Wait(); err != nil { + return err + } + + logging.LogMessage(io, "Resetting admin user password") + { + res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{ + Password: ts.Config.TriplestoreAdminPassword, + AppSettings: TriplestoreUserAppSettings{ + DefaultInference: true, + DefaultVisGraphSchema: true, + DefaultSameas: true, + IgnoreSharedQueries: false, + ExecuteCount: true, + }, + GrantedAuthorities: []string{"ROLE_ADMIN"}, + }, "", "") + if err != nil { + return fmt.Errorf("failed to create triplestore user: %s", err) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + // we set the password => requests are unauthorized + // so we still need to enable security (see below!) + case http.StatusUnauthorized: + // a password is needed => security is already enabled. + // the password may or may not work, but that's a problem for later + logging.LogMessage(io, "Security is already enabled") + return nil + default: + return fmt.Errorf("failed to create triplestore user: %s", err) + } + } + + logging.LogMessage(io, "Enabling Triplestore security") + { + res, err := ts.OpenRaw("POST", "/rest/security", true, "", "") + if err != nil { + return fmt.Errorf("failed to enable triplestore security: %s", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return errTriplestoreFailedSecurity + } + + return nil + } +} diff --git a/internal/config/runtime.go b/internal/config/runtime.go index af7c17d..48db811 100644 --- a/internal/config/runtime.go +++ b/internal/config/runtime.go @@ -1,14 +1,9 @@ package config import ( - "embed" "path/filepath" ) -// Runtime contains runtime resources to be installed into any instance -//go:embed all:runtime -var Runtime embed.FS - // RuntimeDir returns the path to the runtime directory func (cfg Config) RuntimeDir() string { return filepath.Join(cfg.DeployRoot, "runtime") diff --git a/internal/wisski/component.go b/internal/wisski/component.go index af780f3..ad87cf1 100644 --- a/internal/wisski/component.go +++ b/internal/wisski/component.go @@ -88,6 +88,11 @@ func (dis *Distillery) Installables() []component.Installable { return getComponents[component.Installable](dis) } +// Installables returns all components that can be installed +func (dis *Distillery) Updateable() []component.Updatable { + return getComponents[component.Updatable](dis) +} + func getComponents[C component.Component](dis *Distillery) (result []C) { all := dis.ComponentsX()