Make 'system_update' more generic

internal/component/component.go

@@ -32,20 +32,6 @@ type Component interface {
 	Base() *ComponentBase
 }
 
-// Installable implements an installable component.
-type Installable interface {
-	Component
-
-	// Stack can be used to gain access to the "docker compose" stack.
-	//
-	// This should internally call [ComponentBase.MakeStack]
-	Stack(env environment.Environment) StackWithResources
-
-	// Context returns a new InstallationContext to be used during installation from the command line.
-	// Typically this should just pass through the parent, but might perform other tasks.
-	Context(parent InstallationContext) InstallationContext
-}
-
 // ComponentBase implements base functionality for a component
 type ComponentBase struct {
 	Core        // the core of the associated distillery

internal/component/installable.go

@@ -1,120 +1,33 @@
 package component
 
 import (
-	"io/fs"
-	"path/filepath"
-
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
-	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
-	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
-	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 )
 
-// TODO: Move this package into components
+// Installable implements an installable component.
+type Installable interface {
+	Component
 
-// StackWithResources represents a Stack that can be automatically installed from a set of resources.
-// See the [Install] method.
-type StackWithResources struct {
-	Stack
-
-	// Installable enabled installing several resources from a (potentially embedded) filesystem.
+	// Stack can be used to gain access to the "docker compose" stack.
 	//
-	// The Resources holds these, with appropriate resources specified below.
-	// These all refer to paths within the Resource filesystem.
-	Resources   fs.FS
-	ContextPath string            // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
-	EnvPath     string            // the '.env' template, will be installed using [unpack.InstallTemplate].
-	EnvContext  map[string]string // context when instantiating the '.env' template
+	// This should internally call [ComponentBase.MakeStack]
+	Stack(env environment.Environment) StackWithResources
 
-	CopyContextFiles []string // Files to copy from the installation context
-
-	MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
-	MakeDirs     []string    // directories to ensure that exist
-
-	TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
+	// Context returns a new InstallationContext to be used during installation from the command line.
+	// Typically this should just pass through the parent, but might perform other tasks.
+	Context(parent InstallationContext) InstallationContext
 }
 
-// InstallationContext is a context to install data in
-type InstallationContext map[string]string
+// Updatable represents a component with an Update method.
+type Updatable interface {
+	Component
 
-// Install installs or updates this stack into the directory specified by stack.Stack().
-//
-// Installation is non-interactive, but will provide debugging output onto io.
-// InstallationContext
-func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
-	if is.ContextPath != "" {
-		// setup the base files
-		if err := unpack.InstallDir(
-			env,
-			is.Dir,
-			is.ContextPath,
-			is.Resources,
-			func(dst, src string) {
-				io.Printf("[install] %s\n", dst)
-			},
-		); err != nil {
-			return err
-		}
-	}
-
-	// configure .env
-	envDest := filepath.Join(is.Dir, ".env")
-	if is.EnvPath != "" && is.EnvContext != nil {
-		io.Printf("[config]  %s\n", envDest)
-		if err := unpack.InstallTemplate(
-			env,
-			envDest,
-			is.EnvContext,
-			is.EnvPath,
-			is.Resources,
-		); err != nil {
-			return err
-		}
-	}
-
-	// make sure that certain dirs exist
-	for _, name := range is.MakeDirs {
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		io.Printf("[make]    %s\n", dst)
-		if is.MakeDirsPerm == fs.FileMode(0) {
-			is.MakeDirsPerm = environment.DefaultDirPerm
-		}
-		if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
-			return err
-		}
-	}
-
-	// copy files from the context!
-	for _, name := range is.CopyContextFiles {
-		// find the source!
-		src, ok := context[name]
-		if !ok {
-			return errors.Errorf("Missing file from context: %q", src)
-		}
-
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		// copy over file from context
-		io.Printf("[copy]    %s (from %s)\n", dst, src)
-		if err := fsx.CopyFile(env, dst, src); err != nil {
-			return errors.Wrapf(err, "Unable to copy file %s", src)
-		}
-	}
-
-	// make sure that certain files exist
-	for _, name := range is.TouchFiles {
-		// find the destination!
-		dst := filepath.Join(is.Dir, name)
-
-		io.Printf("[touch]   %s\n", dst)
-		if err := fsx.Touch(env, dst); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	// Update updates or initializes the provided components.
+	// It is called after the component has been installed (if applicable).
+	//
+	// It may send output to the provided stream.
+	//
+	// Updating should be idempotent, meaning running it multiple times must not break the existing system.
+	Update(stream stream.IOStream) error
 }

internal/component/instances/runtime.go

@@ -0,0 +1,29 @@
+package instances
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var errBootstrapFailedRuntime = exit.Error{
+	Message:  "failed to update runtime",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Runtime contains runtime resources to be installed into any instance
+//go:embed all:runtime
+var runtimeResources embed.FS
+
+// Update installs or updates runtime components needed by this component.
+func (instances *Instances) Update(stream stream.IOStream) error {
+	err := unpack.InstallDir(instances.Core.Environment, instances.Config.RuntimeDir(), "runtime", runtimeResources, func(dst, src string) {
+		stream.Printf("[copy]  %s\n", dst)
+	})
+	if err != nil {
+		return errBootstrapFailedRuntime.Wrap(err)
+	}
+	return nil
+}

internal/component/instances/runtime/README

@@ -0,0 +1,2 @@
+Files in this folder are utility scripts to be used from within individual WissKI instances. 
+They are mounted under runtime/ and should be used with care. 

internal/component/instances/runtime/blind_update.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# This utility script can be used to blindly update all dependencies to their latest versions. 
+# It does not perform any checking whatsoever. 
+
+# update the main modules
+cd /var/www/data/project || exit 1
+chmod u+rw web/sites/default/
+composer update
+
+# update the db
+drush -y updatedb
+
+# update the wisski dependencies
+cd /var/www/data/project/web/modules/contrib/wisski || exit 1
+composer update

internal/component/instances/runtime/create_admin.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+set -e
+
+# read user
+USER=$1
+if [ -z "$USER" ]; then
+    echo "Usage: create_admin.sh USERNAME"
+    exit 1
+fi
+
+# read password
+echo "Enter Password for $USER:"
+read -s PASS
+echo "Enter the same password again:"
+read -s PASS2
+
+if [ "$PASS" != "$PASS2" ]; then
+    echo "Passwords not equal"
+    exit 1
+fi;
+
+# create the user and add the admin role
+cd /var/www/data/project/
+drush user:create "$USER" --password="$PASS"
+drush user-add-role administrator "$USER"

internal/component/instances/runtime/cron.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# This utility script can be used to run all cron tasks. 
+
+cd /var/www/data/project || exit 1
+export PATH=/var/www/data/project/vendor/bin:$PATH
+
+drush core-cron

internal/component/instances/runtime/install_colorbox.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+# make a temporary directory and cd into it
+TEMPDIR=$(mktemp -d)
+pushd "$TEMPDIR"
+
+# curl the colorbox zip and unpack it
+curl -L https://github.com/jackmoore/colorbox/archive/master.zip --output master.zip
+unzip master.zip
+
+# make the directory for libraries, and remove the old colorbox installation
+chmod u+rw /var/www/data/project/web/sites/default/
+mkdir -p /var/www/data/project/web/sites/default/libraries/
+rm -rf /var/www/data/project/web/sites/default/libraries/colorbox
+
+# copy over the new installation
+mv colorbox-master/ /var/www/data/project/web/sites/default/libraries/colorbox
+
+# cleanup
+popd
+rm -rf "$TEMPDIR"

internal/component/instances/runtime/patch_easyrdf.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# This script can be used to repatch EasyRDF when needed. 
+cd /var/www/data/project/web/modules/contrib/wisski || exit 1
+EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
+patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"

internal/component/instances/runtime/patch_triples.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# This script can be used to repatch EasyRDF when needed. 
+cd /var/www/data/project/web/modules/contrib/wisski/ || exit 1
+TRIPLESTABCONTROLLER="./wisski_adapter_sparql11_pb/src/Controller/Sparql11TriplesTabController.php"
+patch -N "$TRIPLESTABCONTROLLER" < "/patch/triples.patch"

internal/component/instances/runtime/use_wisski.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+# read user
+VERSION=$1
+if [ -z "$VERSION" ]; then
+    echo "Usage: use_wisski.sh VERSION"
+    exit 1
+fi
+
+# update the main modules
+cd /var/www/data/project
+chmod u+rw web/sites/default/
+composer require "drupal/wisski:$VERSION"
+
+# update the wisski dependencies
+pushd /var/www/data/project/web/modules/contrib/wisski
+composer update
+popd
+
+# update the db
+drush -y updatedb

internal/component/instances/runtime/wisski_2x_3x.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+# temporarily extend permissions
+chmod 777 web/sites/default
+chmod 666 web/sites/default/*settings.php
+chmod 666 web/sites/default/*services.yml
+
+# update the core itself
+composer require 'drupal/internal/core-recommended:^9' 'drupal/internal/core-composer-scaffold:^9' 'drupal/internal/core-project-message:^9' --update-with-dependencies --no-update
+composer update
+composer require 'drupal/wisski'
+
+# update requirements for wisski!
+pushd web/modules/contrib/wisski || exit 1
+composer update
+popd || exit 1
+
+# run the update and clear the cache!
+drush updatedb --yes
+# drush cc
+
+# and reset everything back to normal
+chmod 755 web/sites/default
+chmod 644 web/sites/default/*settings.php
+chmod 644 web/sites/default/*services.yml

internal/component/sql/update.go

@@ -14,8 +14,8 @@ var errSQLUnableToCreateUser = errors.New("unable to create administrative user"
 var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name")
 var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
 
-// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
-func (sql *SQL) Bootstrap(io stream.IOStream) error {
+// Update initializes or updates the SQL database.
+func (sql *SQL) Update(io stream.IOStream) error {
 	if err := sql.WaitShell(); err != nil {
 		return err
 	}

internal/component/stack.go

@@ -4,9 +4,13 @@ package component
 import (
 	"bufio"
 	"bytes"
-	"errors"
+	"io/fs"
+	"path/filepath"
 
 	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
+	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
 )
 
@@ -183,3 +187,109 @@ func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
 	}
 	return ds.Env.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
 }
+
+// StackWithResources represents a Stack that can be automatically installed from a set of resources.
+// See the [Install] method.
+type StackWithResources struct {
+	Stack
+
+	// Installable enabled installing several resources from a (potentially embedded) filesystem.
+	//
+	// The Resources holds these, with appropriate resources specified below.
+	// These all refer to paths within the Resource filesystem.
+	Resources   fs.FS
+	ContextPath string            // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
+	EnvPath     string            // the '.env' template, will be installed using [unpack.InstallTemplate].
+	EnvContext  map[string]string // context when instantiating the '.env' template
+
+	CopyContextFiles []string // Files to copy from the installation context
+
+	MakeDirsPerm fs.FileMode // permission for diretories, defaults to [environment.DefaultDirCreate]
+	MakeDirs     []string    // directories to ensure that exist
+
+	TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
+}
+
+// InstallationContext is a context to install data in
+type InstallationContext map[string]string
+
+// Install installs or updates this stack into the directory specified by stack.Stack().
+//
+// Installation is non-interactive, but will provide debugging output onto io.
+// InstallationContext
+func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
+	if is.ContextPath != "" {
+		// setup the base files
+		if err := unpack.InstallDir(
+			env,
+			is.Dir,
+			is.ContextPath,
+			is.Resources,
+			func(dst, src string) {
+				io.Printf("[install] %s\n", dst)
+			},
+		); err != nil {
+			return err
+		}
+	}
+
+	// configure .env
+	envDest := filepath.Join(is.Dir, ".env")
+	if is.EnvPath != "" && is.EnvContext != nil {
+		io.Printf("[config]  %s\n", envDest)
+		if err := unpack.InstallTemplate(
+			env,
+			envDest,
+			is.EnvContext,
+			is.EnvPath,
+			is.Resources,
+		); err != nil {
+			return err
+		}
+	}
+
+	// make sure that certain dirs exist
+	for _, name := range is.MakeDirs {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[make]    %s\n", dst)
+		if is.MakeDirsPerm == fs.FileMode(0) {
+			is.MakeDirsPerm = environment.DefaultDirPerm
+		}
+		if err := env.MkdirAll(dst, is.MakeDirsPerm); err != nil {
+			return err
+		}
+	}
+
+	// copy files from the context!
+	for _, name := range is.CopyContextFiles {
+		// find the source!
+		src, ok := context[name]
+		if !ok {
+			return errors.Errorf("Missing file from context: %q", src)
+		}
+
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		// copy over file from context
+		io.Printf("[copy]    %s (from %s)\n", dst, src)
+		if err := fsx.CopyFile(env, dst, src); err != nil {
+			return errors.Wrapf(err, "Unable to copy file %s", src)
+		}
+	}
+
+	// make sure that certain files exist
+	for _, name := range is.TouchFiles {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[touch]   %s\n", dst)
+		if err := fsx.Touch(env, dst); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/component/triplestore/database.go

@@ -3,15 +3,12 @@ package triplestore
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
 
-	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/wait"
 	"github.com/pkg/errors"
-	"github.com/tkw1536/goprogram/stream"
 )
 
 type TriplestoreUserPayload struct {
@@ -148,59 +145,3 @@ type Repository struct {
 	Writable   bool   `json:"writable"`
 	Local      bool   `json:"local"`
 }
-
-var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
-
-func (ts Triplestore) Bootstrap(io stream.IOStream) error {
-	logging.LogMessage(io, "Waiting for Triplestore")
-	if err := ts.Wait(); err != nil {
-		return err
-	}
-
-	logging.LogMessage(io, "Resetting admin user password")
-	{
-		res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
-			Password: ts.Config.TriplestoreAdminPassword,
-			AppSettings: TriplestoreUserAppSettings{
-				DefaultInference:      true,
-				DefaultVisGraphSchema: true,
-				DefaultSameas:         true,
-				IgnoreSharedQueries:   false,
-				ExecuteCount:          true,
-			},
-			GrantedAuthorities: []string{"ROLE_ADMIN"},
-		}, "", "")
-		if err != nil {
-			return fmt.Errorf("failed to create triplestore user: %s", err)
-		}
-		defer res.Body.Close()
-
-		switch res.StatusCode {
-		case http.StatusOK:
-			// we set the password => requests are unauthorized
-			// so we still need to enable security (see below!)
-		case http.StatusUnauthorized:
-			// a password is needed => security is already enabled.
-			// the password may or may not work, but that's a problem for later
-			logging.LogMessage(io, "Security is already enabled")
-			return nil
-		default:
-			return fmt.Errorf("failed to create triplestore user: %s", err)
-		}
-	}
-
-	logging.LogMessage(io, "Enabling Triplestore security")
-	{
-		res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
-		if err != nil {
-			return fmt.Errorf("failed to enable triplestore security: %s", err)
-		}
-		defer res.Body.Close()
-
-		if res.StatusCode != http.StatusOK {
-			return errTriplestoreFailedSecurity
-		}
-
-		return nil
-	}
-}

internal/component/triplestore/update.go

@@ -0,0 +1,66 @@
+package triplestore
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
+
+func (ts Triplestore) Update(io stream.IOStream) error {
+	logging.LogMessage(io, "Waiting for Triplestore")
+	if err := ts.Wait(); err != nil {
+		return err
+	}
+
+	logging.LogMessage(io, "Resetting admin user password")
+	{
+		res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
+			Password: ts.Config.TriplestoreAdminPassword,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{"ROLE_ADMIN"},
+		}, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+		defer res.Body.Close()
+
+		switch res.StatusCode {
+		case http.StatusOK:
+			// we set the password => requests are unauthorized
+			// so we still need to enable security (see below!)
+		case http.StatusUnauthorized:
+			// a password is needed => security is already enabled.
+			// the password may or may not work, but that's a problem for later
+			logging.LogMessage(io, "Security is already enabled")
+			return nil
+		default:
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+	}
+
+	logging.LogMessage(io, "Enabling Triplestore security")
+	{
+		res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to enable triplestore security: %s", err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return errTriplestoreFailedSecurity
+		}
+
+		return nil
+	}
+}