Do a large chunk of the move to go

This commit moves a huge chunk of the code to go. The TODO.md document
indicates what is left to be done.

.gitignore

@@ -1,3 +1,4 @@
+/wdcli
 /distillery/overrides.json
 authorized_keys
 .vagrant

.tool-versions

@@ -0,0 +1 @@
+golang 1.18.5

TODO.md

@@ -0,0 +1,52 @@
+# WissKI-Distillery in Go
+
+This document describes the process of moving the distillery to using golang for the control plane (outside of docker containers).
+
+## Bootstrapping
+
+This documents the bootstraping process.
+Work in progress.
+
+- `wdcli bootstrap $DIRECTORY`
+    0. Create the deployment directory
+    1. Copy over the executable (unless it already exists)
+    2. Create a default configuration file (unless it already exists)
+    3. Store the directory in a file called .wdcli in the $HOME directory
+
+- `wdcli system_update`
+    - to be documented
+## Future Work
+
+- Move `provision_entrypoint.sh` into go
+- Avoid running `docker compose` executable and shift it to a library
+- Automatically bootstrap the docker container sql connection (use proper environment variables)
+- Make error handling consistent
+- Add a server that serves information
+- Migrate the individual commands below
+- restructure resource files
+- Documentation
+
+## Migrating Individual Commands
+- [ ] backup_all.sh
+- [ ] backup_instance.sh
+- [x] blind_update.sh
+- [x] blind_update_all.sh
+- [x] cron_all.sh
+- [x] info.sh
+- [x] ls.sh
+- [x] make_mysql_account.sh
+- [ ] monday_full.sh
+- [ ] monday_short.sh
+- [x] mysql.sh
+- [x] provision.sh
+- [x] purge.sh
+- [x] rebuild.sh
+- [x] rebuild_all.sh
+- [x] reserve.sh
+- [x] shell.sh
+- [x] system_install.sh
+- [x] system_update.sh
+- [x] update_prefix_config.sh
+
+## TO BE REMOVED
+- [ ] call_update_php_hack.sh

cmd/blind_update.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// BlindUpdate is the 'blind-update' command
+var BlindUpdate wisski_distillery.Command = blindUpdate{}
+
+type blindUpdate struct {
+	Force       bool `short:"f" long:"force" description:"force running blind-update even if AutoBlindUpdate is set to false"`
+	Positionals struct {
+		Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run blind-update in"`
+	} `positional-args:"true"`
+}
+
+func (blindUpdate) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "blind_update",
+		Description: "Runs the blind update in the provided instances",
+	}
+}
+
+var errBlindUpdateFailed = exit.Error{
+	Message:  "Failed to run blind update script for instance %q: exited with code %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (bu blindUpdate) Run(context wisski_distillery.Context) error {
+	instances, err := context.Environment.Instances(bu.Positionals.Slug...)
+	if err != nil {
+		return err
+	}
+
+	for _, instance := range instances {
+		if !(instance.IsBlindUpdateEnabled() || bu.Force) {
+			context.EPrintf("skipping instance %q\n", instance.Slug)
+			continue
+		}
+		context.EPrintf("Updating instance %s\n", instance.Slug)
+
+		code := instance.Shell(context.IOStream, "/utils/blind_update.sh")
+		if code != 0 {
+			return errBlindUpdateFailed.WithMessageF(instance.Slug, code)
+		}
+	}
+
+	return nil
+}

cmd/bootstrap.go

@@ -0,0 +1,187 @@
+package cmd
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	cfg "github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/hostname"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/password"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Bootstrap is the 'bootstrap' command
+var Bootstrap wisski_distillery.Command = bootstrap{}
+
+type bootstrap struct {
+	Directory string `short:"r" long:"root-directory" description:"path to the root deployment directory" default:"/var/www/deploy"`
+	Hostname  string `short:"h" long:"hostname" description:"default hostname of the distillery (default: system hostname)"`
+}
+
+func (bootstrap) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: false,
+		},
+		Command:     "bootstrap",
+		Description: "Bootstraps the installation of a Distillery System",
+	}
+}
+
+var errBootstrapDifferent = exit.Error{
+	Message:  "refusing to bootstrap: base directory is already set to %s.",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBootstrapFailedToCreateDirectory = exit.Error{
+	Message:  "failed to create directory %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBootstrapFailedToSaveDirectory = exit.Error{
+	Message:  "failed to register base directory: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBoostrapFailedToCopyExe = exit.Error{
+	Message:  "failed to copy wdcli executable: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBootstrapWriteConfig = exit.Error{
+	Message:  "failed to write configuration file: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBootstrapOpenConfig = exit.Error{
+	Message:  "failed to open configuration file: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errBootstrapCreateFile = exit.Error{
+	Message:  "failed to touch configuration file: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (bs bootstrap) Run(context wisski_distillery.Context) error {
+	root := bs.Directory
+
+	// check that we didn't get a different base directory
+	{
+		got, err := env.ReadBaseDirectory()
+		if err == nil && got != "" && got != root {
+			return errBootstrapDifferent.WithMessageF(got)
+		}
+	}
+
+	{
+		logging.LogMessage(context.IOStream, "Creating root deployment directory")
+		if err := os.MkdirAll(root, fs.ModeDir); err != nil {
+			return errBootstrapFailedToCreateDirectory.WithMessageF(root)
+		}
+		if err := env.WriteBaseDirectory(root); err != nil {
+			return errBootstrapFailedToSaveDirectory.WithMessageF(root)
+		}
+		context.Println(root)
+	}
+
+	// TODO: Read these from the command line?
+	wdcliPath := filepath.Join(root, "wdcli")
+	envPath := filepath.Join(root, ".env")
+	domain := bs.Hostname
+	if domain == "" {
+		domain = hostname.FQDN()
+	}
+	overridesPath := filepath.Join(root, "overrides.json")
+	authorizedKeysFile := filepath.Join(root, "authorized_keys")
+
+	{
+		logging.LogMessage(context.IOStream, "Copying over wdcli executable")
+		exe, err := os.Executable()
+		if err != nil {
+			return errBoostrapFailedToCopyExe.WithMessageF(err)
+		}
+
+		err = fsx.CopyFile(wdcliPath, exe)
+		if err != nil && err != fsx.ErrCopySameFile {
+			return errBoostrapFailedToCopyExe.WithMessageF(err)
+		}
+		context.Println(wdcliPath)
+	}
+
+	{
+		if !fsx.IsFile(envPath) {
+			if err := logging.LogOperation(func() error {
+				password, err := password.Password(128)
+				if err != nil {
+					return errBootstrapWriteConfig.WithMessageF(err)
+				}
+
+				if err := distillery.InstallTemplate(envPath, filepath.Join("resources", "templates", "bootstrap", "env"), map[string]string{
+					"DEPLOY_ROOT":          root,
+					"DEFAULT_DOMAIN":       domain,
+					"SELF_OVERRIDES_FILE":  overridesPath,
+					"AUTHORIZED_KEYS_FILE": authorizedKeysFile,
+
+					"GRAPHDB_ADMIN_USER":     "admin",
+					"GRAPHDB_ADMIN_PASSWORD": password[:64],
+
+					"MYSQL_ADMIN_USER":     "admin",
+					"MYSQL_ADMIN_PASSWORD": password[64:],
+				}); err != nil {
+					return errBootstrapWriteConfig.WithMessageF(err)
+				}
+
+				return nil
+			}, context.IOStream, "Installing configuration file"); err != nil {
+				return err
+			}
+
+			if err := logging.LogOperation(func() error {
+
+				context.Println(overridesPath)
+				if err := distillery.InstallTemplate(overridesPath, filepath.Join("resources", "templates", "bootstrap", "overrides.json"), map[string]string{}); err != nil {
+					return errBootstrapCreateFile.WithMessageF(err)
+				}
+
+				context.Println(authorizedKeysFile)
+				if err := distillery.InstallTemplate(authorizedKeysFile, filepath.Join("resources", "templates", "bootstrap", "global_authorized_keys"), map[string]string{}); err != nil {
+					return errBootstrapCreateFile.WithMessageF(err)
+				}
+
+				return nil
+			}, context.IOStream, "Creating additional config files"); err != nil {
+				return err
+			}
+		}
+
+	}
+
+	// re-read the configuration and print it!
+	logging.LogMessage(context.IOStream, "Configuration is now complete")
+	f, err := os.Open(envPath)
+	if err != nil {
+		return errBootstrapOpenConfig.WithMessageF(err)
+	}
+	defer f.Close()
+
+	var config cfg.Config
+	if err := config.Unmarshal(f); err != nil {
+		return errBootstrapOpenConfig.WithMessageF(err)
+	}
+	context.Println(config)
+
+	// Tell the user how to proceed
+	logging.LogMessage(context.IOStream, "Bootstrap is complete")
+	context.Printf("Adjust the configuration file at %s\n", envPath)
+	context.Printf("Then grab a GraphDB zipped source file and run:\n")
+	context.Printf("%s system_update /path/to/graphdb.zip\n", wdcliPath)
+
+	return nil
+}

cmd/config.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+)
+
+// Config is the configuration command
+var Config wisski_distillery.Command = config{}
+
+type config struct {
+}
+
+func (s config) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "config",
+		Description: "Prints information about configuration",
+	}
+}
+
+func (s config) Run(context wisski_distillery.Context) error {
+	context.Printf("%#v", context.Environment.Config)
+	return nil
+}

cmd/cron.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Cron is the 'cron' command
+var Cron wisski_distillery.Command = cron{}
+
+type cron struct {
+	Positionals struct {
+		Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run cron in"`
+	} `positional-args:"true"`
+}
+
+func (cron) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "cron",
+		Description: "Runs the cron script for several instances",
+	}
+}
+
+var errCronFailed = exit.Error{
+	Message:  "Failed to run cron script for instance %q: exited with code %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (cr cron) Run(context wisski_distillery.Context) error {
+	instances, err := context.Environment.Instances(cr.Positionals.Slug...)
+	if err != nil {
+		return err
+	}
+
+	// iterate over the instances and store the last value of error
+	for _, instance := range instances {
+		logging.LogOperation(func() error {
+			code := instance.Shell(context.IOStream, "/utils/cron.sh")
+			if code != 0 {
+				// keep going, because we want to run as many crons as possible
+				err = errBlindUpdateFailed.WithMessageF(instance.Slug, code)
+				context.EPrintln(err)
+			}
+
+			return nil
+		}, context.IOStream, "running cron for instance %s", instance.Slug)
+	}
+
+	return err
+}

cmd/info.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+)
+
+// Info is then 'info' command
+var Info wisski_distillery.Command = info{}
+
+type info struct {
+	Positionals struct {
+		Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show info about"`
+	} `positional-args:"true"`
+}
+
+func (info) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "info",
+		Description: "Provide information about a single repository",
+	}
+}
+
+func (i info) Run(context wisski_distillery.Context) error {
+	instance, err := context.Environment.Instance(i.Positionals.Slug)
+	if err != nil {
+		return err
+	}
+
+	context.Printf("URL:                  %s\n", instance.URL())
+	context.Printf("Base directory:       %s\n", instance.FilesystemBase)
+
+	context.Printf("SQL Database:         %s\n", instance.SqlDatabase)
+	context.Printf("SQL Username:         %s\n", instance.SqlUser)
+	context.Printf("SQL Password:         %s\n", instance.SqlPassword)
+
+	context.Printf("GraphDB Repository:   %s\n", instance.GraphDBRepository)
+	context.Printf("GraphDB Username:     %s\n", instance.GraphDBUser)
+	context.Printf("GraphDB Password:     %s\n", instance.GraphDBPassword)
+
+	return nil
+}

cmd/license.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/legal"
+)
+
+// License is the 'wdcli license' command.
+//
+// The license command prints to standard output legal notices about the wdcli program.
+var License wisski_distillery.Command = license{}
+
+type license struct{}
+
+func (license) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: false,
+		},
+		Command:     "license",
+		Description: "Print licensing information about wdcli and exit. ",
+	}
+}
+
+func (license) AfterParse() error {
+	return nil
+}
+
+func (license) Run(context wisski_distillery.Context) error {
+	context.Printf(stringLicenseInfo, wisski_distillery.License, legal.Notices)
+	return nil
+}
+
+const stringLicenseInfo = `
+wdcli -- WissKI Distillery Command Line Utility
+https://github.com/FAU-CDI/wisski-distillery
+
+================================================================================
+wdcli is licensed under the terms of the AGPL Version 3.0 License:
+
+%s
+================================================================================
+
+Furthermore, this executable may include code from the following projects:
+%s
+`

cmd/ls.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+)
+
+// Ls is the 'ls' command
+var Ls wisski_distillery.Command = ls{}
+
+type ls struct {
+	Positionals struct {
+		Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug(s) of instance(s) to list"`
+	} `positional-args:"true"`
+}
+
+func (ls) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "ls",
+		Description: "Lists WissKI instances",
+	}
+}
+
+func (l ls) Run(context wisski_distillery.Context) error {
+	instances, err := context.Environment.Instances(l.Positionals.Slug...)
+	if err != nil {
+		return err
+	}
+
+	for _, instance := range instances {
+		context.Println(instance.Slug)
+	}
+
+	return nil
+}

cmd/make_mysql_account.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"fmt"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/sqle"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/parser"
+)
+
+// Shell is the 'shell' command
+var MakeMysqlAccount wisski_distillery.Command = makeMysqlAccount{}
+
+type makeMysqlAccount struct{}
+
+func (makeMysqlAccount) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		ParserConfig: parser.Config{
+			IncludeUnknown: true,
+		},
+		Command:     "make_mysql_account",
+		Description: "Open a shell in the provided instance",
+	}
+}
+
+var errUnableToReadUsername = exit.Error{
+	ExitCode: exit.ExitGeneric,
+	Message:  "unable to read username: %s",
+}
+
+var errUnableToReadPassword = exit.Error{
+	ExitCode: exit.ExitGeneric,
+	Message:  "unable to read password: %s",
+}
+
+func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error {
+	context.Printf("Username>")
+	username, err := context.ReadLine()
+	if err != nil {
+		return errUnableToReadUsername.WithMessageF(err)
+	}
+
+	context.Printf("Password>")
+	password, err := context.ReadPassword()
+	if err != nil {
+		return errUnableToReadPassword.WithMessageF(err)
+	}
+
+	query := sqle.Format("CREATE USER ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username)
+	if err != nil {
+		return err
+	}
+	code := context.Environment.SQLShell(context.IOStream, "-e", query)
+
+	if code != 0 {
+		return exit.Error{
+			ExitCode: exit.ExitCode(uint8(code)),
+			Message:  fmt.Sprintf("Exit code %d", code),
+		}
+	}
+	return nil
+}

cmd/mysql.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"fmt"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/parser"
+)
+
+// Mysql is the 'mysql' command
+var Mysql wisski_distillery.Command = mysql{}
+
+type mysql struct {
+	Positionals struct {
+		Args []string `positional-arg-name:"ARGS" description:"arguments to pass to the mysql command"`
+	} `positional-args:"true"`
+}
+
+func (mysql) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		ParserConfig: parser.Config{
+			IncludeUnknown: true,
+		},
+		Command:     "mysql",
+		Description: "Opens a mysql shell",
+	}
+}
+
+func (ms mysql) Run(context wisski_distillery.Context) error {
+	code := context.Environment.SQLShell(context.IOStream, ms.Positionals.Args...)
+	if code != 0 {
+		return exit.Error{
+			ExitCode: exit.ExitCode(uint8(code)),
+			Message:  fmt.Sprintf("Exit code %d", code),
+		}
+	}
+	return nil
+}

cmd/prefix.go

@@ -0,0 +1,77 @@
+package cmd
+
+import (
+	"io/fs"
+	"os"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Cron is the 'cron' command
+var UpdatePrefixConfig wisski_distillery.Command = updateprefixconfig{}
+
+type updateprefixconfig struct{}
+
+func (updateprefixconfig) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "update_prefix_config",
+		Description: "Updates the prefix configuration",
+	}
+}
+
+var errPrefixUpdateFailed = exit.Error{
+	Message:  "Failed to update the prefix configuration: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+
+	instances, err := dis.AllInstances()
+	if err != nil {
+		return errPrefixUpdateFailed.WithMessageF(err)
+	}
+
+	target := dis.ResolverPrefixConfig()
+
+	// print the configuration
+	config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm)
+	if err != nil {
+		return errPrefixUpdateFailed.WithMessageF(err)
+	}
+
+	// iterate over the instances and store the last value of error
+	for _, instance := range instances {
+		if err := logging.LogOperation(func() error {
+			// read the prefix config
+			data, err := instance.PrefixConfig()
+			if err != nil {
+				return err
+			}
+			context.IOStream.Printf("%s", data)
+
+			// and write it out!
+			if _, err := config.WriteString(data); err != nil {
+				return err
+			}
+
+			return nil
+		}, context.IOStream, "reading prefix config %s", instance.Slug); err != nil {
+			return errPrefixUpdateFailed.WithMessageF(err)
+		}
+	}
+
+	// and restart the resolver to apply the config!
+	logging.LogMessage(context.IOStream, "restarting resolver stack")
+	if err := dis.ResolverStack().Restart(context.IOStream); err != nil {
+		return errPrefixUpdateFailed.WithMessageF(err)
+	}
+
+	return err
+}

cmd/provision.go

@@ -0,0 +1,121 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Provision is the 'provision' command
+var Provision wisski_distillery.Command = provision{}
+
+type provision struct {
+	Positionals struct {
+		Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to create"`
+	} `positional-args:"true"`
+}
+
+func (provision) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "provision",
+		Description: "Creates a new WissKI Instance",
+	}
+}
+
+// TODO: AfterParse to check instance!
+
+var errProvisionAlreadyExists = exit.Error{
+	Message:  "Instance %q already exists",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errProvisionGeneric = exit.Error{
+	Message:  "Unable to provision instance %s: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (p provision) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+	slug := p.Positionals.Slug
+
+	// check that it doesn't already exist
+	logging.LogMessage(context.IOStream, "Provisioning new WissKI instance %s", slug)
+	if exists, err := dis.HasInstance(slug); err != nil || exists {
+		return errProvisionAlreadyExists.WithMessageF(slug)
+	}
+
+	// make it in-memory
+	instance, err := dis.NewInstance(slug)
+	if err != nil {
+		return errProvisionGeneric.WithMessageF(slug, err)
+	}
+
+	// check that the base directory does not exist
+	logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
+	if fsx.IsDirectory(instance.FilesystemBase) {
+		return errProvisionAlreadyExists.WithMessageF(slug)
+	}
+
+	// Store in bookkeeping
+	if err := logging.LogOperation(func() error {
+		if err := instance.Update(); err != nil {
+			return errProvisionGeneric.WithMessageF(slug, err)
+		}
+
+		return nil
+	}, context.IOStream, "Updating bookkeeping database"); err != nil {
+		return err
+	}
+
+	// create the sql
+	if err := logging.LogOperation(func() error {
+		if err := dis.SQLProvision(instance.SqlDatabase, instance.SqlUser, instance.SqlPassword); err != nil {
+			return errProvisionGeneric.WithMessageF(slug, err)
+		}
+
+		return nil
+	}, context.IOStream, "Provisioning SQL Database"); err != nil {
+		return err
+	}
+
+	// create the triplestore
+	if err := logging.LogOperation(func() error {
+		if err := dis.TriplestoreProvision(instance.GraphDBRepository, instance.Domain(), instance.GraphDBUser, instance.GraphDBPassword); err != nil {
+			return errProvisionGeneric.WithMessageF(slug, err)
+		}
+
+		return nil
+	}, context.IOStream, "Provisioning Triplestore"); err != nil {
+		return err
+	}
+
+	// run the provision script
+	if err := logging.LogOperation(func() error {
+		if err := instance.Provision(context.IOStream); err != nil {
+			return errProvisionGeneric.WithMessageF(slug, err)
+		}
+
+		return nil
+	}, context.IOStream, "Running setup scripts"); err != nil {
+		return err
+	}
+
+	// start the container!
+	logging.LogMessage(context.IOStream, "Starting Container")
+	if err := instance.Stack().Up(context.IOStream); err != nil {
+		return err
+	}
+
+	// and we're done!
+	logging.LogMessage(context.IOStream, "Instance has been provisioned")
+	context.Printf("URL:      %s\n", instance.URL().String())
+	context.Printf("Username: %s\n", instance.DrupalUsername)
+	context.Printf("Password: %s\n", instance.DrupalPassword)
+
+	return nil
+}

cmd/purge.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"os"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Provision is the 'provision' command
+var Purge wisski_distillery.Command = purge{}
+
+type purge struct {
+	Yes         bool `short:"y" long:"yes" description:"Skip asking for confirmation"`
+	Positionals struct {
+		Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to purge"`
+	} `positional-args:"true"`
+}
+
+func (purge) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "purge",
+		Description: "Purges a WissKI Instance",
+	}
+}
+
+var errPurgeNoDetails = exit.Error{
+	Message:  "Unable to find instance details for purge: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errPurgeNoConfirmation = exit.Error{
+	Message:  "Aborting after request was not confirmed. Either type `yes` or pass `--yes` on the command line",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (p purge) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+	slug := p.Positionals.Slug
+
+	// check the confirmation from the user
+	if !p.Yes {
+		context.Printf("About to remove repository %s. This cannot be undone.\n", slug)
+		context.Printf("Type 'yes' to continue: ")
+		line, err := context.ReadLine()
+		if err != nil || line != "yes" {
+			return errPurgeNoConfirmation
+		}
+	}
+
+	// load the instance (first via bookkeeping, then via defaults)
+	logging.LogMessage(context.IOStream, "Checking bookkeeping table")
+	instance, err := dis.Instance(slug)
+	if err == env.ErrInstanceNotFound {
+		context.Println("Not found in bookkeeping table, assuming defaults")
+		instance, err = dis.NewInstance(slug)
+	}
+	if err != nil {
+		return errPurgeNoDetails.WithMessageF(err)
+	}
+
+	// remove docker stack
+	logging.LogMessage(context.IOStream, "Stopping and removing docker container")
+	if err := instance.Stack().Down(context.IOStream); err != nil {
+		context.EPrintln(err)
+	}
+
+	// remove the filesystem
+	logging.LogMessage(context.IOStream, "Removing from filesystem %s", instance.FilesystemBase)
+	if err := os.RemoveAll(instance.FilesystemBase); err != nil {
+		context.EPrintln(err)
+	}
+
+	// remove the triplestore
+	logging.LogOperation(func() error {
+		logging.LogMessage(context.IOStream, "Removing user %s", instance.GraphDBUser)
+		if err := dis.TriplestorePurgeUser(instance.GraphDBUser); err != nil {
+			context.EPrintln(err)
+		}
+
+		logging.LogMessage(context.IOStream, "Removing repository %s", instance.GraphDBRepository)
+		if err := dis.TriplestorePurgeRepo(instance.GraphDBRepository); err != nil {
+			context.EPrintln(err)
+		}
+
+		return nil
+	}, context.IOStream, "Removing from Triplestore")
+
+	// remove the sql
+	logging.LogOperation(func() error {
+		logging.LogMessage(context.IOStream, "Removing user %s", instance.SqlUser)
+		if err := dis.SQLPurgeUser(instance.SqlUser); err != nil {
+			context.EPrintln(err)
+		}
+
+		logging.LogMessage(context.IOStream, "Removing database %s", instance.SqlDatabase)
+		if err := dis.SQLPurgeDatabase(instance.SqlDatabase); err != nil {
+			context.EPrintln(err)
+		}
+
+		return nil
+	}, context.IOStream, "Removing from SQL")
+
+	// remove from bookkeeping
+	logging.LogMessage(context.IOStream, "Removing instance from bookkeeping")
+	if err := instance.Delete(); err != nil {
+		context.EPrintln(err)
+	}
+
+	logging.LogMessage(context.IOStream, "Instance %s has been purged", slug)
+	return nil
+}

cmd/rebuild.go

@@ -0,0 +1,65 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Cron is the 'cron' command
+var Rebuild wisski_distillery.Command = rebuild{}
+
+type rebuild struct {
+	Positionals struct {
+		Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run rebuild"`
+	} `positional-args:"true"`
+}
+
+func (rebuild) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "rebuild",
+		Description: "Runs the rebuild script for several instances",
+	}
+}
+
+var errRebuildFailed = exit.Error{
+	Message:  "Failed to run rebuild script for instance %q: exited with code %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (rb rebuild) Run(context wisski_distillery.Context) error {
+	instances, err := context.Environment.Instances(rb.Positionals.Slug...)
+	if err != nil {
+		return err
+	}
+
+	// iterate over the instances and store the last value of error
+	var globalErr error
+	for _, instance := range instances {
+		logging.LogOperation(func() error {
+			s := instance.Stack()
+			if err := logging.LogOperation(func() error {
+				return s.Install(context.IOStream, stack.InstallationContext{})
+			}, context.IOStream, "Installing docker stack"); err != nil {
+				globalErr = err
+				return err
+			}
+
+			if err := logging.LogOperation(func() error {
+				return s.Update(context.IOStream, true)
+			}, context.IOStream, "Updating docker stack"); err != nil {
+				globalErr = err
+				return err
+			}
+
+			return nil
+		}, context.IOStream, "Rebuilding instance %s", instance.Slug)
+	}
+
+	return globalErr
+}

cmd/reserve.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Reserve is the 'reserve' command
+var Reserve wisski_distillery.Command = reserve{}
+
+type reserve struct {
+	Positionals struct {
+		Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to reserve"`
+	} `positional-args:"true"`
+}
+
+func (reserve) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		Command:     "reserve",
+		Description: "Reserves a new WissKI Instance",
+	}
+}
+
+// TODO: AfterParse to check instance!
+
+var errReserveAlreadyExists = exit.Error{
+	Message:  "Instance %q already exists",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errReserveGeneric = exit.Error{
+	Message:  "Unable to provision instance %s: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (r reserve) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+	slug := r.Positionals.Slug
+
+	// check that it doesn't already exist
+	logging.LogMessage(context.IOStream, "Reserving new WissKI instance %s", slug)
+	if exists, err := dis.HasInstance(slug); err != nil || exists {
+		return errProvisionAlreadyExists.WithMessageF(slug)
+	}
+
+	// make it in-memory
+	instance, err := dis.NewInstance(slug)
+	if err != nil {
+		return errProvisionGeneric.WithMessageF(slug, err)
+	}
+
+	// check that the base directory does not exist
+	logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
+	if fsx.IsDirectory(instance.FilesystemBase) {
+		return errProvisionAlreadyExists.WithMessageF(slug)
+	}
+
+	// setup docker stack
+	s := instance.ReserveStack()
+	{
+		if err := logging.LogOperation(func() error {
+			return s.Install(context.IOStream, stack.InstallationContext{})
+		}, context.IOStream, "Installing docker stack"); err != nil {
+			return err
+		}
+
+		if err := logging.LogOperation(func() error {
+			return s.Update(context.IOStream, true)
+		}, context.IOStream, "Updating docker stack"); err != nil {
+			return err
+		}
+	}
+
+	// and we're done!
+	logging.LogMessage(context.IOStream, "Instance has been reserved")
+	context.Printf("URL:      %s\n", instance.URL().String())
+
+	return nil
+}

cmd/shell.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"fmt"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/parser"
+)
+
+// Shell is the 'shell' command
+var Shell wisski_distillery.Command = shell{}
+
+type shell struct {
+	Positionals struct {
+		Slug string   `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show run shell in"`
+		Args []string `positional-arg-name:"ARGS" description:"arguments to pass to the shell"`
+	} `positional-args:"true"`
+}
+
+func (shell) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		ParserConfig: parser.Config{
+			IncludeUnknown: true,
+		},
+		Command:     "shell",
+		Description: "Open a shell in the provided instance",
+	}
+}
+
+func (sh shell) Run(context wisski_distillery.Context) error {
+	instance, err := context.Environment.Instance(sh.Positionals.Slug)
+	if err != nil {
+		return err
+	}
+
+	code := instance.Shell(context.IOStream, sh.Positionals.Args...)
+	if code != 0 {
+		return exit.Error{
+			ExitCode: exit.ExitCode(uint8(code)),
+			Message:  fmt.Sprintf("Exit code %d", code),
+		}
+	}
+	return nil
+}

cmd/system_update.go

@@ -0,0 +1,182 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/FAU-CDI/wisski-distillery/internal/execx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/parser"
+)
+
+// SystemUpdate is the 'system_update' command
+var SystemUpdate wisski_distillery.Command = systemupdate{}
+
+type systemupdate struct {
+	SkipCoreUpdates bool `short:"s" long:"skip-core-updates" description:"Skip applying operating system and other core system updates"`
+	Positionals     struct {
+		GraphdbZip string `positional-arg-name:"PATH_TO_GRAPHDB_ZIP" required:"1-1" description:"path to the graphdb.zip file"`
+	} `positional-args:"true"`
+}
+
+func (systemupdate) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: env.Requirements{
+			NeedsConfig: true,
+		},
+		ParserConfig: parser.Config{
+			IncludeUnknown: true,
+		},
+		Command:     "system_update",
+		Description: "Installs and Update Components of the WissKI Distillery System",
+	}
+}
+
+var errNoGraphDBZip = exit.Error{
+	Message:  "%s does not exist",
+	ExitCode: exit.ExitCommandArguments,
+}
+
+func (s systemupdate) AfterParse() error {
+	_, err := os.Stat(s.Positionals.GraphdbZip)
+	if os.IsNotExist(err) {
+		return errNoGraphDBZip.WithMessageF(s.Positionals.GraphdbZip)
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+var errFailedToCreateDirectory = exit.Error{
+	Message:  "failed to create directory %s: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errFailedRuntime = exit.Error{
+	Message:  "failed to update runtime: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (si systemupdate) Run(context wisski_distillery.Context) error {
+	dis := context.Environment
+
+	// create all the other directories
+	logging.LogMessage(context.IOStream, "Ensuring distillery installation directories exist")
+	for _, d := range []string{
+		dis.Config.DeployRoot,
+		dis.InstancesDir(),
+		dis.InprogressBackupPath(),
+		dis.FinalBackupPath(),
+	} {
+		context.Println(d)
+		if err := os.MkdirAll(d, os.ModeDir); err != nil {
+			return errFailedToCreateDirectory.WithMessageF(d, err)
+		}
+	}
+
+	if !si.SkipCoreUpdates {
+		// install system updates
+		logging.LogMessage(context.IOStream, "Updating Operating System Packages")
+		if err := si.mustExec(context, "", "apt-get", "update"); err != nil {
+			return err
+		}
+		if err := si.mustExec(context, "", "apt-get", "upgrade", "-y"); err != nil {
+			return err
+		}
+
+		// install docker
+		logging.LogMessage(context.IOStream, "Installing / Updating Docker")
+		if err := si.mustExec(context, "", "apt-get", "install", "curl"); err != nil {
+			return err
+		}
+		if err := si.mustExec(context, "", "/bin/sh", "-c", "curl -fsSL https://get.docker.com -o - | /bin/sh"); err != nil {
+			return err
+		}
+	}
+
+	// create the docker network
+	// TODO: Use docker API for this
+	logging.LogMessage(context.IOStream, "Updating Docker Configuration")
+	si.mustExec(context, "", "docker", "network", "create", "distillery")
+
+	// install and update the various stacks!
+	ctx := stack.InstallationContext{
+		"graphdb.zip": si.Positionals.GraphdbZip,
+	}
+
+	if err := logging.LogOperation(func() error {
+		for _, stack := range dis.Stacks() {
+			if err := logging.LogOperation(func() error {
+				return stack.Install(context.IOStream, ctx)
+			}, context.IOStream, "Installing docker stack %q", stack.Name); err != nil {
+				return err
+			}
+
+			if err := logging.LogOperation(func() error {
+				return stack.Update(context.IOStream, true)
+			}, context.IOStream, "Updating docker stack %q", stack.Name); err != nil {
+				return err
+			}
+		}
+		return nil
+	}, context.IOStream, "Updating Components"); err != nil {
+		return err
+	}
+
+	if err := logging.LogOperation(func() error {
+		if err := distillery.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), func(dst, src string) {
+			context.Printf("[copy]  %s\n", dst)
+		}); err != nil {
+			return errFailedRuntime.WithMessageF(err)
+		}
+		return nil
+	}, context.IOStream, "Unpacking Runtime Components"); err != nil {
+		return err
+	}
+
+	if err := logging.LogOperation(func() error {
+		if err := dis.SQLBootstrap(context.IOStream); err != nil {
+			return err
+		}
+		return nil
+	}, context.IOStream, "Bootstraping SQL database"); err != nil {
+		return err
+	}
+
+	if err := logging.LogOperation(func() error {
+		if err := dis.TriplestoreBootstrap(context.IOStream); err != nil {
+			return err
+		}
+		return nil
+	}, context.IOStream, "Bootstraping Triplestore"); err != nil {
+		return err
+	}
+
+	logging.LogMessage(context.IOStream, "System has been updated")
+	return nil
+}
+
+var errMustExecFailed = exit.Error{
+	Message: "process exited with code %d",
+}
+
+// mustExec indicates that the given executable process must complete successfully.
+// If it does not, returns errMustExecFailed
+func (si systemupdate) mustExec(context wisski_distillery.Context, workdir string, exe string, argv ...string) error {
+	if workdir == "" {
+		workdir = context.Environment.Config.DeployRoot
+	}
+	code := execx.Exec(context.IOStream, workdir, exe, argv...)
+	if code != 0 {
+		err := errMustExecFailed.WithMessageF(code)
+		err.ExitCode = exit.ExitCode(code)
+		return err
+	}
+	return nil
+}

cmd/wdcli/main.go

@@ -0,0 +1,110 @@
+// Command wdcli implement the entry point for the wisski-distillery
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime/debug"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/cmd"
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var wdcli = wisski_distillery.NewProgram()
+
+func init() {
+	// self commands
+	wdcli.Register(cmd.Config)
+	wdcli.Register(cmd.License)
+
+	// setup commands
+	wdcli.Register(cmd.Bootstrap)
+	wdcli.Register(cmd.SystemUpdate)
+
+	// sql commands
+	wdcli.Register(cmd.Mysql)
+	wdcli.Register(cmd.MakeMysqlAccount)
+
+	// instance setup and teardown
+	wdcli.Register(cmd.Provision)
+	wdcli.Register(cmd.Purge)
+	wdcli.Register(cmd.Reserve)
+	wdcli.Register(cmd.Rebuild)
+
+	// instance management
+	wdcli.Register(cmd.Ls)
+	wdcli.Register(cmd.Info)
+
+	// instance tasks
+	wdcli.Register(cmd.Shell)
+	wdcli.Register(cmd.BlindUpdate)
+	wdcli.Register(cmd.Cron)
+	wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration
+
+	// backup & cron
+	// wdcli.Register(cmd.BackupInstance)
+	// wdcli.Register(cmd.BackupAll)
+}
+
+// an error when no arguments are provided.
+var errNoArgumentsProvided = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Need at least one argument. Use `wdcli license` to view licensing information. ",
+}
+
+func main() {
+	// recover from calls to panic(), and exit the program appropriatly.
+	// This has to be in the main() function because any of the library functions might be broken.
+	// For this reason, as few ggman functions as possible are used here; just stuff from the top-level ggman package.
+	defer func() {
+		if err := recover(); err != nil {
+			fmt.Fprintf(os.Stderr, fatalPanicMessage, err)
+			debug.PrintStack()
+			exit.ExitPanic.Return()
+		}
+	}()
+
+	streams := stream.FromEnv()
+
+	// when there are no arguments then parsing argument *will* fail
+	//
+	// we don't need to even bother with the rest of the program
+	// just immediatly return a custom error message.
+	if len(os.Args) == 1 {
+		streams.Die(errNoArgumentsProvided)
+		errNoArgumentsProvided.Return()
+		return
+	}
+
+	// creat a new set of parameters
+	// and then use them to execute the main command
+	err := func() error {
+		params, err := env.ParamsFromEnv()
+		if err != nil {
+			return streams.Die(err)
+		}
+
+		return wdcli.Main(streams, params, os.Args[1:])
+	}()
+
+	// return the error to the user
+
+	exit.AsError(err).Return()
+}
+
+const fatalPanicMessage = `Fatal Error: Panic
+
+The wdcli program panicked and had to abort execution. This is usually
+indicative of a bug. If this occurs repeatedly you might want to consider
+filing an issue in the issue tracker at:
+
+https://github.com/FAU-CDI/wisski-distillery/issues
+
+Below is debug information that might help the developers track down what
+happened.
+
+panic: %v
+`

distillery/blind_update.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-require_slug_argument
-
-
-# if the site doesn't exist, I can't open a shell. 
-if ! sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' does not exist in bookeeping table. "
-    echo "I can't rebuild it. "
-    exit 1
-fi;
-
-# Read everything from the database
-read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
-
-# cd into the right directory
-cd "$INSTANCE_BASE_DIR"
-
-# and open a www-data shell
-docker-compose exec barrel /user_shell.sh /utils/blind_update.sh

distillery/blind_update_all.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-DISABLE_LOG=0
-
-# update all the instances
-for slug in $(sql_bookkeep_list_updateable); do
-    log_info "=> /bin/bash $DIR/blind_update.sh '$slug'"
-    /bin/bash "$DIR/blind_update.sh" "$slug";
-done
-

distillery/cron_all.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-unset DISABLE_LOG
-
-# update all the instances
-for slug in $(sql_bookkeep_list); do
-    read -r INSTANCE_BASE_DIR <<< "$(sql_bookkeep_load "${slug}" "filesystem_base" | tail -n +2)"
-    log_info "=> Runnning cron for '$slug'"
-    cd "$INSTANCE_BASE_DIR"
-    docker-compose exec barrel /bin/bash /utils/cron.sh
-done
-

distillery/info.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-DISABLE_LOG=0
-require_slug_argument
-
-
-# if the site doesn't exist, I can't open a shell. 
-if ! sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' does not exist in bookeeping table. "
-    echo "I can't show info about it. "
-    exit 1
-fi;
-
-# Read everything from the database
-read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER GRAPHDB_PASSWORD <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user,graphdb_password" | tail -n +2)"
-
-GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
-
-# read sql configuration
-cd "$INSTANCE_BASE_DIR"
-read -r  SQL_DATABASE SQL_USER SQL_PASS SQL_OTHER <<< "$(docker-compose exec barrel drush sql:conf --format=tsv --show-passwords)"
-
-echo "=================================================================================="
-echo "URL:                  http://$INSTANCE_DOMAIN"
-echo "Base directory:       ${INSTANCE_BASE_DIR}"
-log_info " => Your GraphDB details (for WissKI Salz) are: "
-echo "Read URL:             http://triplestore:7200/repositories/$GRAPHDB_REPO"
-echo "Write URL:            http://triplestore:7200/repositories/$GRAPHDB_REPO/statements"
-echo "Username:             $GRAPHDB_USER"
-echo "Password:             $GRAPHDB_PASSWORD"
-echo "Authorization Header: $GRAPHDB_HEADER"
-echo "Writable:             yes"
-echo "Default Graph URI:    http://$INSTANCE_DOMAIN/#"
-echo "Ontology Paths:       (empty)"
-echo "SameAs property:      http://www.w3.org/2002/07/owl#sameAs"
-log_info " => Your SQL detsils are: "
-echo "SQL Database:         $SQL_DATABASE"
-echo "SQL Username:         $SQL_USER"
-echo "SQL Password:         $SQL_PASS"

distillery/ls.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-DISABLE_LOG=0
-
-# list all the existing instances
-sql_bookkeep_list

distillery/make_mysql_account.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh
-DISABLE_LOG=0
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-
-# wait for sql to come up
-wait_for_sql > /dev/null
-
-echo "Creating new MySQL user with root privileges. "
-read -p 'Enter Username:' MYSQL_USER
-read -sp 'Enter password:' MYSQL_PASSWORD
-
-if ! is_valid_slug "$MYSQL_USER"; then
-    echo "Not a valid username: ${MYSQL_USER}"
-    echo "User must be alphanumeric for sql injection reasons. "
-    echo "You can always create a user manually. "
-    exit 1
-fi
-
-if ! is_valid_slug "$MYSQL_PASSWORD"; then
-    echo "Not a valid password: ${MYSQL_PASSWORD}"
-    echo "Password must be alphanumeric for sql injection reasons. "
-    echo "You can always create a user manually. "
-    exit 1
-fi
-
-dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO \`${MYSQL_USER}\`@\`%\` WITH GRANT OPTION; FLUSH PRIVILEGES;"
-
-log_info "Created user ${MYSQL_USER}"

distillery/mysql.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-
-# wait for sql to come up
-wait_for_sql > /dev/null
-dockerized_mysql_interactive "$@"

distillery/provision.sh

@@ -1,106 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-require_slug_argument
-
-# wait for sql to be awake
-wait_for_sql
-
-# check if the site exists
-if sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' already exists in bookeeping table. "
-    echo "Refusing to work"
-    exit 1;
-fi
-
-# Randomly generate the database name and user we will configure. 
-# Use the 'randompw' alias for this. 
-log_info " => Generating new MySQL password"
-MYSQL_PASSWORD="$(randompw)"
-
-# Initialize the SQL database with those credentials. 
-log_info " => Intializing new SQL database '${MYSQL_DATABASE}' and user '$MYSQL_USER'. "
-dockerized_mysql -e "CREATE DATABASE \`${MYSQL_DATABASE}\`;"
-dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}';"
-dockerized_mysql -e "GRANT ALL PRIVILEGES ON \`${MYSQL_DATABASE}\`.* TO \`${MYSQL_USER}\`@\`%\`;"
-dockerized_mysql -e "FLUSH PRIVILEGES;"
-
-# Create a new repository for GraphDB. 
-# Use the template for this.
-log_info " => Generating new GraphDB repository '$GRAPHDB_REPO'"
-load_template "repository/graphdb-repo.ttl" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "INSTANCE_DOMAIN" "${INSTANCE_DOMAIN}" | \
-curl -X POST $GRAPHDB_AUTH_FLAGS \
-    http://127.0.0.1:7200/rest/repositories \
-    --header 'Content-Type: multipart/form-data' \
-    -F "config=@-"
-
-# Generate a random password for the GraphDB user
-log_info " => Generating a new GraphDB password"
-GRAPHDB_PASSWORD="$(randompw)"
-
-# Create the user and grant them access to the creatd database. 
-log_info " => Creating GraphDB user '$GRAPHDB_USER'"
-load_template "repository/graphdb-user.json" "GRAPHDB_USER" "${GRAPHDB_USER}" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "GRAPHDB_PASSWORD" "${GRAPHDB_PASSWORD}" | \
-curl -X POST $GRAPHDB_AUTH_FLAGS \
-    "http://127.0.0.1:7200/rest/security/users/${GRAPHDB_USER}" \
-    --header 'Content-Type: application/json' \
-    --header 'Accept: text/plain' \
-    -d @-
-
-log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'"
-mkdir -p "$INSTANCE_BASE_DIR"
-mkdir -p "$INSTANCE_DATA_DIR"
-mkdir -p "$INSTANCE_DATA_DIR/.composer"
-mkdir -p "$INSTANCE_DATA_DIR/data"
-touch "$INSTANCE_DATA_DIR/authorized_keys"
-chmod a+rw "$INSTANCE_DATA_DIR/authorized_keys"
-
-# Generate some more random credentials, this time for drupal. 
-# We again make use of the randompw alias. 
-log_info " => Generating new drupal credentials"
-DRUPAL_USER="admin"
-DRUPAL_PASS="$(randompw)"
-
-# TODO: copy over docker-compose into the right directory
-log_info " => Creating instance directory"
-install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR"
-
-# Log all the details into the bookeeping database
-log_info "=> Storing configuration in bookkeeping table"
-sql_bookkeep_insert \
-    "slug,filesystem_base,sql_database,sql_user,sql_password,graphdb_repository,graphdb_user,graphdb_password" \
-    "\"${SLUG}\",\"${INSTANCE_BASE_DIR}\",\"${MYSQL_DATABASE}\",\"${MYSQL_USER}\",\"${MYSQL_PASSWORD}\",\"${GRAPHDB_REPO}\",\"${GRAPHDB_USER}\",\"${GRAPHDB_PASSWORD}\""
-
-log_info " => Writing configuration file"
-load_template "docker-env/barrel" \
-    "REAL_PATH" "${INSTANCE_DATA_DIR}" \
-    "GLOBAL_AUTHORIZED_KEYS_FILE" "${GLOBAL_AUTHORIZED_KEYS_FILE}" \
-    "VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \
-    "SLUG" "${SLUG}" \
-    "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
-    "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
-    "DISTILLERY_DIR" "${DIR}" \
-    > "$INSTANCE_BASE_DIR/.env"
-
-
-log_info " => Running and building image"
-cd "$INSTANCE_BASE_DIR"
-docker-compose build --pull
-docker-compose pull
-
-log_info " => Running provision script"
-docker-compose run --rm barrel /bin/bash -c "sudo PATH=\$PATH -u www-data /bin/bash /provision_container.sh \
-    \"${INSTANCE_DOMAIN}\" \
-    \"${MYSQL_DATABASE}\" \"${MYSQL_USER}\" \"${MYSQL_PASSWORD}\" \
-    \"${GRAPHDB_REPO}\" \"${GRAPHDB_USER}\" \"${GRAPHDB_PASSWORD}\" \
-    \"${DRUPAL_USER}\" \"${DRUPAL_PASS}\" \
-    \"${DRUPAL_VERSION}\" \"${WISSKI_VERSION}\""
-
-
-log_info " => Starting container"
-docker-compose up -d
-

distillery/purge.sh

@@ -1,68 +0,0 @@
-# To install a new system:
-
-# This script will provision a new Drupal instance and make it available to apache. 
-# Usage: sudo ./provision.sh $SLUG
-# In case the installation fails, it will bail out and leave you with an incomplete installation. 
-# To delete an incomplete installation, use the ./purge.sh script, or try fixing the error manually. 
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-require_slug_argument
-
-# wait for sql to be awake
-wait_for_sql
-
-while true; do
-    log_info " => I'm about to delete the '$SLUG' site from this system. "
-    read -p "This can not be undone. Please type 'y' to continue: " yn
-    case $yn in
-        [Yy]* ) break;;
-        * ) echo "Abort. "; exit 1;;
-    esac
-done
-
-# check if the site exists
-if ! sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' does not exist in bookeeping table. "
-    echo "I'll try to cleanup with the current defaults. "
-    echo "This may or may not work. "
-else
-    # Read all the configuration from the database
-    log_info " => Reading components from database"
-    read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
-fi
-
-# stop the running system container
-if [ -d "$INSTANCE_BASE_DIR" ] ; then
-    log_info "=> Stopping running system"
-    cd "$INSTANCE_BASE_DIR"
-    docker-compose down -v || true
-fi;
-
-cd
-
-# delete the mysql database. 
-log_info " => Deleting MySQL database '$MYSQL_DATABASE' and user '$MYSQL_USER'. "
-dockerized_mysql -e "DROP DATABASE IF EXISTS \`${MYSQL_DATABASE}\`;"
-dockerized_mysql -e "DROP USER IF EXISTS \`${MYSQL_USER}\`@\`%\`;"
-dockerized_mysql -e "FLUSH PRIVILEGES;"
-
-# Clear the GraphDB repository. 
-log_info " => Deleting GraphDB repository '$GRAPHDB_REPO'"
-curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/repositories/$GRAPHDB_REPO
-
-log_info " => Deleting GraphDB user '$GRAPHDB_USER'"
-curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/security/users/$GRAPHDB_USER
-
-
-# Delete the directory
-log_info " => Deleting '$INSTANCE_BASE_DIR'"
-rm -rf "$INSTANCE_BASE_DIR"
-
-log_info " => Clearing bookkeeping record"
-sql_bookeep_delete "$SLUG" || true
-
-log_info " => '$SLUG' has been purged. "

distillery/rebuild.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-require_slug_argument
-
-
-# if the site doesn't exist, I can't open a shell. 
-if ! sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' does not exist in bookeeping table. "
-    echo "I can't rebuild it. "
-    exit 1
-fi;
-
-# Read everything from the database
-read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
-
-log_info " => Touching authorized_keys file"
-touch "$INSTANCE_BASE_DIR/data/authorized_keys"
-chmod a+rw "$INSTANCE_BASE_DIR/data/authorized_keys"
-
-log_info " => Updating compose files"
-install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR"
-
-log_info "=> Rebuilding and restarting '$INSTANCE_BASE_DIR'"
-update_stack "$INSTANCE_BASE_DIR"

distillery/rebuild_all.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-DISABLE_LOG=0
-
-# update all the instances
-for slug in $(sql_bookkeep_list); do
-    log_info "=> /bin/bash $DIR/rebuild.sh '$slug'"
-    /bin/bash "$DIR/rebuild.sh" "$slug";
-done
-

distillery/reserve.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-require_slug_argument
-
-# wait for sql to be awake
-wait_for_sql
-
-# check if the site exists
-if sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' already exists in bookeeping table. "
-    echo "Refusing to work"
-    exit 1;
-fi
-
-log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'"
-mkdir -p "$INSTANCE_BASE_DIR"
-install_resource_dir "compose/reserve" "$INSTANCE_BASE_DIR"
-
-log_info " => Writing configuration file"
-load_template "docker-env/reserve" \
-    "VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \
-    "SLUG" "${SLUG}" \
-    "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
-    "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
-    > "$INSTANCE_BASE_DIR/.env"
-
-
-log_info " => Running and building image"
-cd "$INSTANCE_BASE_DIR"
-docker-compose build --pull
-docker-compose pull
-
-log_info " => Starting container"
-docker-compose up -d
-
-log_info " => $INSTANCE_DOMAIN has been reserved"

distillery/resources.go

@@ -0,0 +1,120 @@
+// TODO: Rename this to resources oncen finished
+package distillery
+
+import (
+	"embed"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+// resourceEmbed contains all the resources required by the WissKI-Distillery package.
+//go:embed all:resources
+var resourceEmbed embed.FS
+
+// InstallResource install a resource src into dest.
+// When it encounters a directory, recursively installs the directory is called.
+// For each installation item, onInstallFile is called, unless onInstallFile is nil.
+//
+// If src points to a file, dst must either be an existing file, or not exist.
+// If src points to a directory, dst must either be an existing directory, or not exist.
+func InstallResource(dst, src string, onInstallFile func(dst, src string)) error {
+	return installFile(dst, resourceEmbed, src, onInstallFile)
+}
+
+var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory")
+var errExpectedDirectoryButGotFile = errors.New("Expected a directory, but got a file")
+
+func installFile(dst string, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
+	// call the on-install file path
+	if onInstallFile != nil {
+		onInstallFile(dst, src)
+	}
+
+	// open the source file!
+	srcFile, err := fsys.Open(src)
+	if err != nil {
+		return errors.Wrapf(err, "Error opening source file %s", src)
+	}
+	defer srcFile.Close()
+
+	// stat the source file to install
+	srcStat, srcErr := srcFile.Stat()
+	if srcErr != nil {
+		return errors.Wrapf(srcErr, "Error calling stat on source %s", src)
+	}
+
+	// if it is a directory, we should recurse!
+	if srcStat.IsDir() {
+		return installDir(dst, srcStat, srcFile, fsys, src, onInstallFile)
+	}
+
+	// determine if we need to create the destination file, or if it already exists
+	dstStat, dstErr := os.Stat(dst)
+	flag := os.O_WRONLY
+	switch {
+	case os.IsNotExist(dstErr):
+		flag |= os.O_CREATE
+	case dstErr != nil:
+		return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
+	case dstStat.IsDir():
+		return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
+	}
+
+	// Open the file
+	dstFile, err := os.OpenFile(dst, flag, srcStat.Mode())
+	if err != nil {
+		return errors.Wrapf(err, "Error opening destination %s", dst)
+	}
+	defer dstFile.Close()
+
+	// copy over the content
+	_, err = io.Copy(dstFile, srcFile)
+	return errors.Wrapf(err, "Error writing to destination %s", dst)
+
+}
+
+func installDir(dst string, srcStat fs.FileInfo, srcFile fs.File, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
+	// make sure it is a directory!
+	dir, ok := srcFile.(fs.ReadDirFile)
+	if !ok {
+		return errExpectedDirectoryButGotFile
+	}
+
+	// create the destination
+	dstStat, dstErr := os.Stat(dst)
+	switch {
+	case os.IsNotExist(dstErr):
+		if err := os.MkdirAll(dst, srcStat.Mode()); err != nil {
+			return errors.Wrapf(err, "Error creating destination directory %s", dst)
+		}
+	case dstErr != nil:
+		return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
+	case !dstStat.IsDir():
+		return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
+	case dstErr == nil:
+	}
+
+	// read the directory
+	entries, err := dir.ReadDir(-1)
+	if err != nil {
+		return errors.Wrapf(err, "Error reading source directory %s", srcFile)
+	}
+
+	// iterate over all the children
+	for _, entry := range entries {
+		if err := func(dst, src string) error {
+			return installFile(dst, fsys, src, onInstallFile)
+		}(
+			filepath.Join(dst, entry.Name()),
+			filepath.Join(src, entry.Name()),
+		); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

distillery/resources/compose/barrel/.env.sample

@@ -3,8 +3,8 @@
 #######################
 
 # Real path for volumes to be stored
-REAL_PATH=/var/www/example.slug
+REAL_PATH=/var/www/deploy/instances/example.slug
-DISTILLERY_DIR=/distillery/
+UTILS_DIR=/var/www/deploy/runtime/utils/
 
 #######################
 ### Web Server settings

distillery/resources/compose/barrel/Dockerfile

@@ -92,7 +92,7 @@ VOLUME /var/www/data
 # Add and configure the entrypoint
 ADD scripts/entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT [ "/entrypoint.sh" ]
+ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
 CMD ["apache2-foreground"]
 
 # Add the provision script and WissKI utils

distillery/resources/compose/barrel/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       - ${REAL_PATH}/.composer:/var/www/.composer
       - ${REAL_PATH}/data:/var/www/data
       - ${REAL_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
-      - ${DISTILLERY_DIR}/utils:/utils:ro
+      - ${UTILS_DIR}:/utils:ro
 
 networks:
   default:

distillery/resources/compose/barrel/scripts/provision_container.sh

@@ -170,3 +170,5 @@ function printdetails() {
     echo "Password:             $DRUPAL_PASS"
 }
 printdetails
+
+exit 0

distillery/resources/compose/self/.env.sample

@@ -1,19 +0,0 @@
-#######################
-# Meta Settings
-#######################
-# The target path to redirect to
-TARGET=https://github.com/FAU-CDI/wisski-distillery
-
-# path to .json
-SELF_OVERRIDES_FILE=/overrides.json
-
-#######################
-### Web Server settings
-#######################
-# the hostname for the website
-VIRTUAL_HOST=example.com
-
-# optional letsencrypt support
-# when blank, ignore
-LETSENCRYPT_HOST=
-LETSENCRYPT_EMAIL=

distillery/resources/templates/bootstrap/env

@@ -1,11 +1,11 @@
 # Several docker-compose files are created to manage global services and the system itself. 
 # On top of this all real-system space will be created under this directory.  
-DEPLOY_ROOT=/var/www/deploy
+DEPLOY_ROOT=${DEPLOY_ROOT}
 
 # Each created Drupal Instance corresponds to a single domain name. 
 # These domain names should either be a complete domain name or a sub-domain of a default domain. 
 # This setting configures the default domain-name to create subdomains of. 
-DEFAULT_DOMAIN=localhost.kwarc.info
+DEFAULT_DOMAIN=${DEFAULT_DOMAIN}
 
 # By default, the default domain redirects to the distillery repository. 
 # If you want to change this, set an alternate domain name here. 
@@ -17,7 +17,7 @@ SELF_EXTRA_DOMAINS=
 
 # You can override individual URLS in the homepage. 
 # Do this by adding URLs (without trailing '/'s) into a JSON file
-SELF_OVERRIDES_FILE=/distillery/overrides.json
+SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
 
 # The system can support setting up certificate(s) automatically. 
 # It can be enabled by setting an email for certbot certificates. 
@@ -52,7 +52,12 @@ DISTILLERY_BOOKKEEPING_TABLE=distillery
 PASSWORD_LENGTH=64
 
 # A file to be used for global authorized_keys for the ssh server.
-GLOBAL_AUTHORIZED_KEYS_FILE=/distillery/authorized_keys
+GLOBAL_AUTHORIZED_KEYS_FILE=${AUTHORIZED_KEYS_FILE}
 
-# The admin password of the GraphDB interface, to be used for queries
+# The admin user and password of the GraphDB interface, to be used for queries
-GRAPHDB_ADMIN_PASSWORD=root
+GRAPHDB_ADMIN_USER=${GRAPHDB_ADMIN_USER}
+GRAPHDB_ADMIN_PASSWORD=${GRAPHDB_ADMIN_PASSWORD}
+
+# The admin password to use for access to mysql
+MYSQL_ADMIN_USER=${MYSQL_ADMIN_USER}
+MYSQL_ADMIN_PASSWORD=${MYSQL_ADMIN_PASSWORD}

distillery/resources/templates/bootstrap/global_authorized_keys

@@ -0,0 +1,2 @@
+# This file contains authorized_keys files valid for every repository in the distillery.
+# To add a key, add one file per line.

distillery/resources/templates/bootstrap/overrides.json

@@ -0,0 +1 @@
+{}

distillery/resources/templates/docker-env/barrel

@@ -1,5 +1,5 @@
 REAL_PATH=${REAL_PATH}
-DISTILLERY_DIR=${DISTILLERY_DIR}
+UTILS_DIR=${UTILS_DIR}
 
 SLUG=${SLUG}
 VIRTUAL_HOST=${VIRTUAL_HOST}

distillery/resources_template.go

@@ -0,0 +1,112 @@
+package distillery
+
+import (
+	"io"
+	"io/fs"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+)
+
+var templateRegexp = regexp.MustCompile(`\${[^}]+}`)
+
+// InstallTemplates open the resource src, and installs it into dst.
+// the template resource must fit into memory.
+//
+// For each variable ${THING} inside dest, a key 'THING' must exist in context.
+// Extra or missing template keys are an error.
+func InstallTemplate(dst, src string, context map[string]string) error {
+	bytes, srcMode, err := doTemplate(src, context)
+	if err != nil {
+		return err
+	}
+
+	// determine if we need to create the destination file, or if it already exists
+	dstStat, dstErr := os.Stat(dst)
+	flag := os.O_WRONLY
+	switch {
+	case os.IsNotExist(dstErr):
+		flag |= os.O_CREATE
+	case dstErr != nil:
+		return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
+	case dstStat.IsDir():
+		return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
+	}
+
+	// open and write the destination file
+	dstFile, err := os.OpenFile(dst, flag, srcMode)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to open file %s", dst)
+	}
+	_, err = dstFile.Write(bytes)
+	return errors.Wrapf(err, "Unable to write destination %s", dst)
+}
+
+// ReadTemplate is like InstallTemplate, except that it writes template into a byte slice and returns it.
+func ReadTemplate(src string, context map[string]string) ([]byte, error) {
+	bytes, _, err := doTemplate(src, context)
+	return bytes, err
+}
+
+func doTemplate(src string, context map[string]string) (bytes []byte, mode fs.FileMode, err error) {
+	// open the source file!
+	srcFile, err := resourceEmbed.Open(src)
+	if err != nil {
+		return nil, mode, errors.Wrapf(err, "Error opening source file %s", src)
+	}
+	defer srcFile.Close()
+
+	// stat the source file to install
+	srcStat, srcErr := srcFile.Stat()
+	if srcErr != nil {
+		return nil, mode, errors.Wrapf(srcErr, "Error calling stat on source %s", src)
+	}
+
+	// should not be a directory
+	if srcStat.IsDir() {
+		return nil, mode, errors.Wrapf(errExpectedFileButGotDirectory, "Error calling stat on source %s", src)
+	}
+
+	// read the template and replace
+	templates, err := io.ReadAll(srcFile)
+	if err != nil {
+		return nil, mode, errors.Wrapf(err, "Unable to read src file %s", src)
+	}
+
+	// keep track of context keys that have not been used
+	unuusedContext := make(map[string]struct{}, len(context))
+	for key := range context {
+		unuusedContext[key] = struct{}{}
+	}
+
+	// replace the template regexp
+	// keeping track of unuused errors
+	var hadError error
+	templates = templateRegexp.ReplaceAllFunc(templates, func(b []byte) []byte {
+		name := string(b[2 : len(b)-1]) // remove the leading ${ and trailing }
+		delete(unuusedContext, name)    // mark the key as having been read
+
+		value, ok := context[name]
+		if hadError != nil && !ok {
+			hadError = errors.Errorf("key %s missing in context", name)
+		}
+		return []byte(value)
+	})
+
+	if hadError != nil {
+		return nil, mode, hadError
+	}
+
+	if len(unuusedContext) != 0 {
+		keys := maps.Keys(unuusedContext)
+		slices.Sort(keys)
+		return nil, mode, errors.Errorf("additional keys %s in context", strings.Join(keys, ","))
+	}
+
+	// return the data and the mode!
+	return templates, srcStat.Mode(), nil
+}

distillery/shell.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh and read the slug argument. 
-DISABLE_LOG=1
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-DISABLE_LOG=0
-require_slug_argument
-
-
-# if the site doesn't exist, I can't open a shell. 
-if ! sql_bookkeep_exists "$SLUG"; then
-    log_error "=> Site '$SLUG' does not exist in bookeeping table. "
-    echo "I can't open a shell there. "
-    exit 1
-fi;
-
-# Read everything from the database
-read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
-
-# cd into the right directory
-cd "$INSTANCE_BASE_DIR"
-
-# and open a www-data shell
-docker-compose exec barrel /user_shell.sh

distillery/system_install.sh

@@ -1,133 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-
-# Read the 'GRAPHDB_ZIP' argument from the command line. 
-# If it's not set, throw an error. 
-GRAPHDB_ZIP=$1
-if [ -z "$GRAPHDB_ZIP" ]; then
-    log_error "Usage: system_install.sh GRAPHDB_ZIP"
-    exit 1;
-fi;
-
-
-# print some general info on the screen
-log_info "=> Preparing system to become a WissKI Distillery"
-echo "This script will install or upgrade this system to become a WissKI distillery. "
-echo "It is idempotent and can safely be run multiple times. "
-sleep 5
-
-
-# Install default system upgrades. 
-log_info "=> Installing system updates"
-apt-get update
-apt-get upgrade -y
-
-# install docker dependencies. 
-log_info "=> Installing docker installer dependencies"
-apt-get update
-apt-get install -y curl
-
-# install docker using an automated script. 
-log_info "=> Installing docker"
-curl -fsSL https://get.docker.com -o - | /bin/sh
-
-# install docker-compose dependencies. 
-log_info "=> Install docker-compose installer dependencies"
-apt-get update
-apt-get install -y python3-pip libffi-dev
-
-# install docker-compose. 
-log_info "=> Installing docker-compose"
-pip3 install --upgrade docker-compose
-
-log_info "=> Creating docker-compose directories and files"
-mkdir -p "$DEPLOY_INSTANCES_DIR"
-mkdir -p "$DEPLOY_WEB_DIR"
-mkdir -p "$DEPLOY_SELF_DIR"
-mkdir -p "$DEPLOY_RESOLVER_DIR"
-mkdir -p "$DEPLOY_SSH_DIR"
-mkdir -p "$DEPLOY_TRIPLESTORE_DIR"
-mkdir -p "$DEPLOY_SQL_DIR"
-mkdir -p "$DEPLOY_BACKUP_INPROGRESS_DIR"
-mkdir -p "$DEPLOY_BACKUP_FINAL_DIR"
-
-log_info "=> Creating 'distillery' network"
-docker network create distillery || true
-
-log_info "=> Creating 'docker-compose' files for the 'web'. "
-install_resource_dir "compose/web" "$DEPLOY_WEB_DIR"
-
-log_info " => Writing 'web' configuration file"
-load_template "docker-env/web" \
-    "DEFAULT_HOST" "${DEFAULT_DOMAIN}" \
-    > "$DEPLOY_WEB_DIR/.env"
-
-log_info "=> Creating 'docker-compose' files for the 'self'. "
-install_resource_dir "compose/self" "$DEPLOY_SELF_DIR"
-
-log_info "=> Creating 'docker-compose' files for the 'resolver'. "
-install_resource_dir "compose/resolver" "$DEPLOY_RESOLVER_DIR"
-touch "$DEPLOY_PREFIX_CONFIG"
-
-log_info "=> Creating 'docker-compose' files for the 'ssh'. "
-install_resource_dir "compose/ssh" "$DEPLOY_SSH_DIR"
-
-# setup the lesencrypt host for the default domain
-if [ -n "$LETSENCRYPT_HOST" ]; then
-    LETSENCRYPT_HOST="$SELF_DOMAIN_SPEC"
-fi;
-
-log_info " => Writing 'self' configuration file"
-load_template "docker-env/self" \
-    "VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \
-    "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
-    "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
-    "TARGET" "${SELF_REDIRECT}" \
-    "OVERRIDES_FILE" "${SELF_OVERRIDES_FILE}" \
-    > "$DEPLOY_SELF_DIR/.env"
-
-log_info " => Writing 'resolver' configuration file"
-load_template "docker-env/resolver" \
-    "VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \
-    "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
-    "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
-    "PREFIX_FILE" "${DEPLOY_PREFIX_CONFIG}" \
-    "DEFAULT_DOMAIN" "${DEFAULT_DOMAIN}" \
-    "LEGACY_DOMAIN" "${SELF_EXTRA_DOMAINS}" \
-    > "$DEPLOY_RESOLVER_DIR/.env"
-
-# copy over the directory
-log_info "=> Creating 'docker-compose' files for the 'triplestore'. "
-install_resource_dir "compose/triplestore" "$DEPLOY_TRIPLESTORE_DIR"
-
-# copy the graphdb.zip
-echo "Writing \"$DEPLOY_TRIPLESTORE_DIR/graphdb.zip\""
-cp "$GRAPHDB_ZIP" "$DEPLOY_TRIPLESTORE_DIR/graphdb.zip"
-
-# create data (volume) location
-mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/data/"
-mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/work/"
-mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/logs/"
-
-# copy over the sql resource directory, then ensure the data diretory for sql exists. 
-log_info "=> Creating 'docker-compose' files for the 'sql'. "
-install_resource_dir "compose/sql" "$DEPLOY_SQL_DIR"
-mkdir -p "$DEPLOY_SQL_DIR/data/"
-
-# Run all the updates via system_update.sh
-log_info " => Running 'system_update.sh'"
-bash "$SCRIPT_DIR/system_update.sh"
-
-log_info "=> Waiting for sql to come up"
-wait_for_sql
-
-log_info "=> Creating '$DISTILLERY_BOOKKEEPING_DATABASE' database and '$DISTILLERY_BOOKKEEPING_TABLE' table"
-load_template "bookkeeping/create.sql" "DATABASE" "$DISTILLERY_BOOKKEEPING_DATABASE" "TABLE" "$DISTILLERY_BOOKKEEPING_TABLE" | \
-    dockerized_mysql
-
-log_info "=> System installation finished, ready to distill. "

distillery/system_update.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-set -e
-
-# read the lib/shared.sh
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR"
-source "$DIR/lib/lib.sh"
-
-log_info "=> Rebuilding and restarting 'web' stack"
-update_stack "$DEPLOY_WEB_DIR"
-
-log_info "=> Rebuilding and restarting 'self' stack"
-update_stack "$DEPLOY_SELF_DIR"
-
-# build and start the ssh server
-log_info "=> Rebuilding and restarting 'ssh' stack"
-update_stack "$DEPLOY_SSH_DIR"
-
-# build and start the triplestore
-log_info "=> Rebuilding and restarting 'triplestore' stack"
-update_stack "$DEPLOY_TRIPLESTORE_DIR"
-
-# build and start the triplestore
-log_info "=> Rebuilding and restarting 'sql' stack"
-update_stack "$DEPLOY_SQL_DIR"
-
-log_info " => Updating Prefix Config"
-cd "$DIR"
-bash update_prefix_config.sh
-
-log_info "=> Rebuilding and restarting 'resolver' stack"
-update_stack "$DEPLOY_RESOLVER_DIR"
-
-log_info "=> System up-to-date. "

env/dirs.go

@@ -0,0 +1,23 @@
+package env
+
+import "path/filepath"
+
+func (dis Distillery) BackupDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "backups")
+}
+
+func (dis Distillery) RuntimeDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "runtime")
+}
+
+func (dis Distillery) RuntimeUtilsDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "runtime", "utils")
+}
+
+func (dis Distillery) InprogressBackupPath() string {
+	return filepath.Join(dis.BackupDir(), "inprogress")
+}
+
+func (dis Distillery) FinalBackupPath() string {
+	return filepath.Join(dis.BackupDir(), "final")
+}

env/distillery.go

@@ -0,0 +1,77 @@
+package env
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Distillery represents a running instance for the distillery
+type Distillery struct {
+	Config *config.Config
+}
+
+func (dis Distillery) HTTPSEnabled() bool {
+	return dis.Config.CertbotEmail != ""
+}
+
+// Returns the default virtual host
+func (dis Distillery) DefaultVirtualHost() string {
+	VIRTUAL_HOST := dis.Config.DefaultDomain
+	if len(dis.Config.SelfExtraDomains) > 0 {
+		VIRTUAL_HOST += "," + strings.Join(dis.Config.SelfExtraDomains, ",")
+	}
+	return VIRTUAL_HOST
+}
+
+func (dis Distillery) DefaultLetsencryptHost() string {
+	if !dis.HTTPSEnabled() {
+		return ""
+	}
+	return dis.DefaultVirtualHost()
+}
+
+// Context returns a new Context belonging to this distillery
+func (dis Distillery) Context() context.Context {
+	return context.Background()
+}
+
+var errNoConfigFile = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Configuration File does not exist",
+}
+
+var errOpenConfig = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "error loading configuration file: %s",
+}
+
+// NewDistillery creates a new distillery object from a set of parameters and requirements
+func NewDistillery(params Params, req Requirements) (env *Distillery, err error) {
+	env = &Distillery{}
+
+	// if we don't need to load the config, there is nothing to do
+	if !req.NeedsConfig {
+		return
+	}
+
+	// if there is no no config file, return
+	cfg := params.ConfigFilePath()
+	if cfg == "" {
+		return nil, errNoConfigFile
+	}
+
+	f, err := os.Open(params.ConfigFilePath())
+	if err != nil {
+		return nil, errOpenConfig.WithMessageF(err)
+	}
+	defer f.Close()
+
+	// unmarshal the config
+	env.Config = &config.Config{}
+	err = env.Config.Unmarshal(f)
+	return
+}

env/instances.go

@@ -0,0 +1,359 @@
+package env
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/alessio/shellescape"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+var errNoBookkeeping = exit.Error{
+	Message:  "instance %q does not exist in bookkeeping table",
+	ExitCode: exit.ExitGeneric,
+}
+
+var ErrInstanceNotFound = exit.Error{
+	Message:  "instance not found",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Instance returns the instance of the WissKI Distillery with the provided slug
+func (dis *Distillery) Instance(slug string) (i Instance, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return i, err
+	}
+
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return i, err
+	}
+
+	// find the instance by slug
+	query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
+	switch {
+	case query.Error != nil:
+		return i, errSQL.WithMessageF(query.Error)
+	case query.RowsAffected == 0:
+		return i, ErrInstanceNotFound
+	default:
+		i.dis = dis
+		return i, nil
+	}
+}
+
+// HasInstance checks if the provided instance exists in the bookeeping table
+func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return false, err
+	}
+
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return false, err
+	}
+
+	query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok)
+	if query.Error != nil {
+		return false, errSQL.WithMessageF(query.Error)
+	}
+	return
+}
+
+// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances.
+func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) {
+	if len(slugs) == 0 {
+		return dis.AllInstances()
+	}
+	return dis.InstancesWith(slugs...)
+}
+
+// AllInstances returns all instances of the WissKI Distillery in consistent order.
+//
+// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order.
+func (dis *Distillery) AllInstances() ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table
+	})
+}
+
+// InstancesWith returns all instances where the slug is in the provided list of names.
+// The returned instances are reordered in a consistent order.
+func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table.Where("slug IN ?", slugs)
+	})
+}
+
+// findInstances finds instance objects based on a query in the bookkeeping table
+func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return nil, err
+	}
+
+	// open the bookkeeping table
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return nil, err
+	}
+
+	// prepare a query
+	find := table
+	if order {
+		find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false})
+	}
+	if query != nil {
+		find = query(find)
+	}
+
+	// fetch bookkeeping instances
+	var bks []bookkeeping.Instance
+	find = find.Find(&bks)
+	if find.Error != nil {
+		return nil, errSQL.WithMessageF(find.Error)
+	}
+
+	// make proper instances
+	instances = make([]Instance, len(bks))
+	for i, bk := range bks {
+		instances[i].Instance = bk
+		instances[i].dis = dis
+	}
+
+	return instances, nil
+}
+
+// Instance represents a bookkeeping instance
+type Instance struct {
+	bookkeeping.Instance
+
+	// Credentials for the drupal instance
+	DrupalUsername string
+	DrupalPassword string
+
+	dis *Distillery
+}
+
+// Update updates the bookkeeping table with this instance.
+func (instance *Instance) Update() error {
+	db, err := instance.dis.sqlBkTable(false)
+	if err != nil {
+		return err
+	}
+
+	// it has never been created => we need to create it in the database
+	if instance.Instance.Created.IsZero() {
+		return db.Create(&instance.Instance).Error
+	}
+
+	// Update based on the primary key!
+	return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error
+}
+
+// Delete deletes this instance from the bookkeeping table
+func (instance *Instance) Delete() error {
+	db, err := instance.dis.sqlBkTable(false)
+	if err != nil {
+		return err
+	}
+
+	// doesn't exist => nothing to delete
+	if instance.Instance.Created.IsZero() {
+		return nil
+	}
+
+	// delete it directly
+	return db.Delete(&instance.Instance).Error
+}
+
+// Shell executes a shell command inside the
+func (instance Instance) Shell(io stream.IOStream, argv ...string) int {
+	return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...)
+}
+
+// Domain returns the full domain name of this instance
+func (instance Instance) Domain() string {
+	return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain)
+}
+
+// IfHttps returns value if the distillery has https enabled, the empty string otherwise
+// TODO: Fix this to be in a proper place
+func (dis *Distillery) IfHttps(value string) string {
+	if !dis.HTTPSEnabled() {
+		return ""
+	}
+	return value
+}
+
+// URL returns the public URL of this instance
+func (instance Instance) URL() *url.URL {
+	// setup domain and path
+	url := &url.URL{
+		Host: instance.Domain(),
+		Path: "/",
+	}
+
+	// use http or https scheme depending on if the distillery has it enabled
+	if instance.dis.HTTPSEnabled() {
+		url.Scheme = "https"
+	} else {
+		url.Scheme = "http"
+	}
+
+	return url
+}
+
+// Stack represents a stack representing this instance
+func (instance Instance) Stack() stack.Installable {
+	return stack.Installable{
+		Stack: stack.Stack{
+			Name: "barrel",
+			Dir:  instance.FilesystemBase,
+		},
+		ContextResource: filepath.Join("resources", "compose", "barrel"),
+
+		EnvFileResource: filepath.Join("resources", "templates", "docker-env", "barrel"),
+		EnvFileContext: map[string]string{
+			"REAL_PATH": instance.FilesystemBase,
+
+			"SLUG":         instance.Slug,
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail),
+
+			"UTILS_DIR":                   instance.dis.RuntimeUtilsDir(),
+			"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
+		},
+
+		CopyContextFiles: nil,
+
+		TouchFiles: []string{
+			"authorized_keys",
+		},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs:     []string{"data", ".composer"},
+	}
+}
+
+func (instance Instance) ReserveStack() stack.Installable {
+	return stack.Installable{
+		Stack: stack.Stack{
+			Name: "reserve",
+			Dir:  instance.FilesystemBase,
+		},
+		ContextResource: filepath.Join("resources", "compose", "reserve"),
+
+		EnvFileResource: filepath.Join("resources", "templates", "docker-env", "reserve"),
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail),
+		},
+	}
+}
+
+// Provision provisions an instance, assuming that the required databases already exist.
+func (instance Instance) Provision(io stream.IOStream) error {
+
+	// create the basic st!
+	st := instance.Stack()
+	if err := st.Install(io, stack.InstallationContext{}); err != nil {
+		return err
+	}
+
+	// Pull and build the stack!
+	if err := st.Update(io, false); err != nil {
+		return err
+	}
+
+	provisionParams := []string{
+		instance.Domain(),
+
+		instance.SqlDatabase,
+		instance.SqlUser,
+		instance.SqlPassword,
+
+		instance.GraphDBRepository,
+		instance.GraphDBUser,
+		instance.GraphDBPassword,
+
+		instance.DrupalUsername,
+		instance.DrupalPassword,
+
+		"", // TODO: DrupalVersion
+		"", // TODO: WissKIVersion
+	}
+
+	// escape the parameter
+	for i, param := range provisionParams {
+		provisionParams[i] = shellescape.Quote(param)
+	}
+
+	// figure out the provision script
+	// TODO: Move the provision script into the control plane!
+	provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
+
+	if st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) != 0 {
+		return errors.New("Unable to run provision script")
+	}
+
+	return nil
+}
+
+func (instance *Instance) NoPrefix() bool {
+	return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip"))
+}
+
+var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
+
+// PrefixConfig returns the prefix config belonging to this instance.
+func (instance *Instance) PrefixConfig() (config string, err error) {
+	// if the user requested to skip the prefix, then don't do anything with it!
+	if instance.NoPrefix() {
+		return "", nil
+	}
+
+	var builder strings.Builder
+
+	// domain
+	builder.WriteString(instance.URL().String() + ":")
+	builder.WriteString("\n")
+
+	// default prefixes
+	wu := stream.NewIOStream(&builder, nil, nil, 0)
+	if instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") != 0 {
+		return "", errPrefixExecFailed
+	}
+
+	// custom prefixes
+	prefixPath := filepath.Join(instance.FilesystemBase, "prefixes")
+	if fsx.IsFile(prefixPath) {
+		prefix, err := os.Open(prefixPath)
+		if err != nil {
+			return "", err
+		}
+		defer prefix.Close()
+		if _, err := io.Copy(&builder, prefix); err != nil {
+			return "", err
+		}
+		builder.WriteString("\n")
+	}
+
+	// and done!
+	return builder.String(), nil
+}

env/instances_provision.go

@@ -0,0 +1,96 @@
+package env
+
+import (
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/password"
+	"github.com/pkg/errors"
+)
+
+func (dis *Distillery) InstancesDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "instances")
+}
+
+func (dis *Distillery) InstanceDir(slug string) string {
+	return filepath.Join(dis.InstancesDir(), slug)
+}
+
+func (dis *Distillery) InstanceSQL(slug string) (database, user string) {
+	database = dis.Config.MysqlDatabasePrefix + slug
+	user = dis.Config.MysqlUserPrefix + slug
+	return
+}
+
+func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) {
+	repo = dis.Config.GraphDBRepoPrefix + slug
+	user = dis.Config.GraphDBUserPrefix + slug
+	return
+}
+
+// Password returns a new password
+func (dis *Distillery) NewPassword() (value string, err error) {
+	return password.Password(dis.Config.PasswordLength)
+}
+
+var errInvalidSlug = errors.New("Not a valid slug")
+
+// NewInstance fills the struct for a new distillery instance.
+// It validates that slug is a valid name for an instance.
+//
+// It does not perform any checks if the instance already exists, or does the creation in the database.
+func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
+
+	// make sure that the slug is valid!
+	if _, err := config.IsValidSlug(slug); err != nil {
+		return i, errInvalidSlug
+	}
+
+	// generate sql data
+	sqlPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	sqlDB, sqlUser := dis.InstanceSQL(slug)
+
+	// generate ts data
+	tsPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	tsRepo, tsUser := dis.InstanceGraphDB(slug)
+
+	// generate drupal data
+	drPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	drUser := "admin"
+
+	// make the instance object!
+	instance := bookkeeping.Instance{
+		Slug: slug,
+
+		OwnerEmail:             "",
+		AutoBlindUpdateEnabled: true,
+
+		FilesystemBase: dis.InstanceDir(slug),
+
+		SqlDatabase: sqlDB,
+		SqlUser:     sqlUser,
+		SqlPassword: sqlPassword,
+
+		GraphDBRepository: tsRepo,
+		GraphDBUser:       tsUser,
+		GraphDBPassword:   tsPassword,
+	}
+
+	i.DrupalUsername = drUser
+	i.DrupalPassword = drPassword
+
+	// store the instance in the object and return it!
+	i.Instance = instance
+	i.dis = dis
+	return i, nil
+}

env/params.go

@@ -0,0 +1,92 @@
+package env
+
+import (
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Params are parameters used for initialization of the environment
+type Params struct {
+	BaseDirectory string
+}
+
+// ConfigFilePath returns the path to the configuration file
+func (params Params) ConfigFilePath() string {
+	if params.BaseDirectory == "" {
+		return ""
+	}
+	return filepath.Join(params.BaseDirectory, ".env")
+}
+
+var errUnableToLoadParams = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Unable to configure wdcli environment: %s",
+}
+
+const BaseDirectoryDefault = "/var/www/deploy"
+
+// ParamsFromEnv creates a new set of parameters from the environment.
+// There is no guarantee that the parameters are valid.
+func ParamsFromEnv() (params Params, err error) {
+	// try to read the base directory
+	value, err := ReadBaseDirectory()
+	switch {
+	case os.IsNotExist(err):
+		params.BaseDirectory = BaseDirectoryDefault
+	case err == nil:
+		params.BaseDirectory = value
+	default:
+		return params, errUnableToLoadParams.WithMessageF(err)
+	}
+
+	return params, nil
+}
+
+var baseConfigFile = ".wdcli"
+
+// ReadBaseDirectory reads the base directory from the environment, or an empty string
+func ReadBaseDirectory() (value string, err error) {
+	// find the current user
+	usr, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	// read the base config file!
+	contents, err := os.ReadFile(filepath.Join(usr.HomeDir, baseConfigFile))
+	if err != nil {
+		return "", err
+	}
+
+	// and trim the spaces!
+	value = strings.TrimSpace(string(contents))
+
+	// check that it is actually set!
+	if len(value) == 0 {
+		return "", errors.New("ReadBaseDirectory: Directory is empty")
+	}
+
+	// and return it!
+	return value, nil
+}
+
+// WriteBaseDirectory writes the base directory to the environment, or returns an error
+func WriteBaseDirectory(dir string) error {
+	// find the current user
+	usr, err := user.Current()
+	if err != nil {
+		return err
+	}
+
+	// read the base config file!
+	return os.WriteFile(
+		filepath.Join(usr.HomeDir, baseConfigFile),
+		[]byte(dir),
+		os.ModePerm,
+	)
+}

env/requirements.go

@@ -0,0 +1,24 @@
+package env
+
+import (
+	"github.com/tkw1536/goprogram"
+	"github.com/tkw1536/goprogram/meta"
+)
+
+type Requirements struct {
+	NeedsConfig bool
+}
+
+// AllowsFlag checks if the provided flag may be passed to fullfill this requirement
+// By default it is used only for help page generation, and may be inaccurate.
+func (r Requirements) AllowsFlag(flag meta.Flag) bool {
+	return true
+}
+
+// Validate validates if this requirement is fullfilled for the provided global flags.
+// It should return either nil, or an error of type exit.Error.
+//
+// Validate does not take into account AllowsOption, see ValidateAllowedOptions.
+func (r Requirements) Validate(arguments goprogram.Arguments[struct{}]) error {
+	return nil
+}

env/stack.go

@@ -0,0 +1,30 @@
+package env
+
+import (
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+// Stacks returns the Stacks of this distillery
+func (dis *Distillery) Stacks() []stack.Installable {
+	// TODO: Do we want to cache these stacks?
+	return []stack.Installable{
+		dis.WebStack(),
+		dis.SelfStack(),
+		dis.ResolverStack(),
+		dis.SSHStack(),
+		dis.TriplestoreStack(),
+		dis.SQLStack(),
+	}
+}
+
+// asCoreStack treats the provided stack as a core component of this distillery.
+func (dis *Distillery) asCoreStack(stack stack.Installable) stack.Installable {
+	stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", stack.Name)
+
+	stack.ContextResource = filepath.Join("resources", "compose", stack.Name)
+	stack.EnvFileResource = filepath.Join("resources", "templates", "docker-env", stack.Name)
+
+	return stack
+}

env/stack_resolver.go

@@ -0,0 +1,39 @@
+package env
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+const ResolverPrefixFile = "prefix.cfg"
+
+func (dis *Distillery) ResolverStack() stack.Installable {
+	stack := dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "resolver",
+		},
+
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST":      dis.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  dis.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
+			"PREFIX_FILE":       "", // set below!
+			"DEFAULT_DOMAIN":    dis.Config.DefaultDomain,
+			"LEGACY_DOMAIN":     strings.Join(dis.Config.SelfExtraDomains, ","),
+		},
+
+		TouchFiles: []string{ResolverPrefixFile},
+	})
+	stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, ResolverPrefixFile)
+	return stack
+}
+
+func (dis *Distillery) ResolverStackPath() string {
+	return dis.ResolverStack().Dir
+}
+
+func (dis Distillery) ResolverPrefixConfig() string {
+	return filepath.Join(dis.ResolverStackPath(), ResolverPrefixFile)
+}

env/stack_self.go

@@ -0,0 +1,28 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) SelfStack() stack.Installable {
+	TARGET := "https://github.com/FAU-CDI/wisski-distillery"
+	if dis.Config.SelfRedirect != nil {
+		TARGET = dis.Config.SelfRedirect.String()
+	}
+
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "self",
+		},
+
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST":      dis.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  dis.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
+			"TARGET":            TARGET,
+			"OVERRIDES_FILE":    dis.Config.SelfOverridesFile,
+		},
+	})
+}
+
+func (dis *Distillery) SelfStackPath() string {
+	return dis.SelfStack().Dir
+}

env/stack_sql.go

@@ -0,0 +1,221 @@
+package env
+
+import (
+	"fmt"
+	"io/fs"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/sqle"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+// SQLStack returns the docker stack that handles the sql database.
+func (dis *Distillery) SQLStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "sql",
+		},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			"data",
+		},
+	})
+}
+
+// SQLStackPath returns the path the SQLStack() lives at.
+func (dis *Distillery) SQLStackPath() string {
+	return dis.SQLStack().Dir
+}
+
+// sqlOpen opens a new sql connection to the provided database using the administrative credentials
+func (env Distillery) sqlOpen(database string, config *gorm.Config) (*gorm.DB, error) {
+	sql := mysql.Config{
+		DSN:               fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", env.Config.MysqlAdminUser, env.Config.MysqlAdminPassword, "127.0.0.1:3306", database),
+		DefaultStringSize: 256,
+	}
+
+	db, err := gorm.Open(mysql.New(sql), config)
+	if err != nil {
+		return db, err
+	}
+
+	gdb, err := db.DB()
+	if err != nil {
+		return db, err
+	}
+	gdb.SetMaxIdleConns(0)
+
+	return db, nil
+}
+
+var errSQL = exit.Error{
+	Message:  "error querying sql database: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+// sqlBkTable returns a gorm connection to the bookkeeping database.
+func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) {
+
+	config := &gorm.Config{}
+	if silent {
+		config.Logger = logger.Default.LogMode(logger.Silent)
+	}
+
+	// open the database
+	db, err := dis.sqlOpen(dis.Config.DistilleryBookkeepingDatabase, config)
+	if err != nil {
+		return nil, errSQL.WithMessageF(err)
+	}
+
+	// load the table
+	table := db.Table(dis.Config.DistilleryBookkeepingTable)
+	if table.Error != nil {
+		return nil, errSQL.WithMessageF(err)
+	}
+
+	return table, nil
+}
+
+// SQLShell executes a mysql shell inside the SQLStack.
+func (dis *Distillery) SQLShell(io stream.IOStream, argv ...string) int {
+	return dis.SQLStack().Exec(io, "sql", "mysql", argv...)
+}
+
+var errSQLBootstrap = exit.Error{
+	Message:  "Unable to boostrap SQL: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+const waitSQLInterval = 1 * time.Second
+
+// SQLWaitForShell waits for the sql database to be reachable via a docker-compose shell
+func (dis *Distillery) SQLWaitForShell() error {
+	n := stream.FromNil()
+	return wait.Wait(func() bool {
+		return dis.SQLShell(n, "-e", "show databases;") == 0
+	}, waitSQLInterval, dis.Context())
+}
+
+// SQLWaitForConnection waits for the sql connection to be alive
+func (dis *Distillery) SQLWaitForConnection() error {
+	return wait.Wait(func() bool {
+		_, err := dis.sqlBkTable(true)
+		return err == nil
+	}, waitSQLInterval, dis.Context())
+}
+
+var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
+
+func (dis *Distillery) sqlRaw(query string, args ...interface{}) bool {
+	sql := sqle.Format(query, args...)
+	return dis.SQLShell(stream.FromNil(), "-e", sql) == 0
+}
+
+// SQLProvision provisions a new sql database and user
+func (dis *Distillery) SQLProvision(name, user, password string) error {
+	// wait for the database
+	if err := dis.SQLWaitForShell(); err != nil {
+		return err
+	}
+
+	// it's not a safe database name!
+	if !sqle.IsSafeDatabaseName(name) {
+		return errInvalidDatabaseName
+	}
+
+	// create the database and user!
+	if !dis.sqlRaw("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
+		return errors.New("SQLProvision: Failed to create user")
+	}
+
+	// and done!
+	return nil
+}
+
+var errSQLPurgeUser = exit.Error{
+	Message:  "Unable to delete user",
+	ExitCode: exit.ExitGeneric,
+}
+
+// SQLPurgeUser deletes the specified user from the database
+func (dis *Distillery) SQLPurgeUser(user string) error {
+	if !dis.sqlRaw("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
+		return errSQLPurgeUser
+	}
+
+	return nil
+}
+
+var errSQLPurgeDB = exit.Error{
+	Message:  "Unable to delete database",
+	ExitCode: exit.ExitGeneric,
+}
+
+// SQLPurgeDatabase deletes the specified db from the database
+func (dis *Distillery) SQLPurgeDatabase(db string) error {
+	if !sqle.IsSafeDatabaseName(db) {
+		return errSQLPurgeDB
+	}
+	if !dis.sqlRaw("DROP DATABASE IF EXISTS `" + db + "`") {
+		return errSQLPurgeDB
+	}
+	return nil
+}
+
+// SQLBootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
+func (dis *Distillery) SQLBootstrap(io stream.IOStream) error {
+	if err := dis.SQLWaitForShell(); err != nil {
+		return errSQLBootstrap.WithMessageF(err)
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating administrative user")
+	{
+		username := dis.Config.MysqlAdminUser
+		password := dis.Config.MysqlAdminPassword
+		if !dis.sqlRaw("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
+			return errSQLBootstrap.WithMessageF("Unable to create administrative user")
+		}
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating sql database")
+	{
+		if !sqle.IsSafeDatabaseName(dis.Config.DistilleryBookkeepingDatabase) {
+			return errSQLBootstrap.WithMessageF("Unsafe database name")
+		}
+		createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", dis.Config.DistilleryBookkeepingDatabase)
+		if !dis.sqlRaw(createDBSQL) {
+			return errSQLBootstrap.WithMessageF(createDBSQL)
+		}
+	}
+
+	// wait for the database to come up
+	logging.LogMessage(io, "Waiting for database update to be complete")
+	dis.SQLWaitForConnection()
+
+	// open the database
+	logging.LogMessage(io, "Migrating bookkeeping table")
+	{
+		db, err := dis.sqlBkTable(false)
+		if err != nil {
+			return errSQLBootstrap.WithMessageF(err)
+		}
+
+		if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
+			return errSQLBootstrap.WithMessageF(err)
+		}
+	}
+
+	return nil
+}

env/stack_ssh.go

@@ -0,0 +1,16 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) SSHStack() stack.Installable {
+	// TODO: Ensure that .env is copied if needed
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "sql",
+		},
+	})
+}
+
+func (dis *Distillery) SSHStackPath() string {
+	return dis.SSHStack().Dir
+}

env/stack_triplestore.go

@@ -0,0 +1,263 @@
+package env
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/fs"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (dis *Distillery) TriplestoreStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "triplestore",
+		},
+
+		CopyContextFiles: []string{"graphdb.zip"},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			filepath.Join("data", "data"),
+			filepath.Join("data", "work"),
+			filepath.Join("data", "logs"),
+		},
+	})
+}
+
+func (dis *Distillery) TriplestoreStackPath() string {
+	return dis.TriplestoreStack().Dir
+}
+
+type TriplestoreUserPayload struct {
+	Password           string                     `json:"password"`
+	AppSettings        TriplestoreUserAppSettings `json:"appSettings"`
+	GrantedAuthorities []string                   `json:"grantedAuthorities"`
+}
+type TriplestoreUserAppSettings struct {
+	DefaultInference      bool `json:"DEFAULT_INFERENCE"`
+	DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
+	DefaultSameas         bool `json:"DEFAULT_SAMEAS"`
+	IgnoreSharedQueries   bool `json:"IGNORE_SHARED_QUERIES"`
+	ExecuteCount          bool `json:"EXECUTE_COUNT"`
+}
+
+var errTriplestoreBootstrap = exit.Error{
+	Message:  "Unable to bootstrap Triplestore: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+const triplestoreBaseURL = "http://127.0.0.1:7200"
+const waitTSInterval = 1 * time.Second
+
+// triplestoreCall makes a request to the triplestore.
+//
+// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
+// When bodyName is empty, simply marshal body as application/json
+func (dis *Distillery) triplestoreRequest(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
+	var reader io.Reader
+
+	var contentType string
+
+	// for "PUT" and "POST" we setup a body
+	if method == "PUT" || method == "POST" {
+		if bodyName != "" {
+			buffer := &bytes.Buffer{}
+			writer := multipart.NewWriter(buffer)
+			contentType = writer.FormDataContentType()
+
+			part, err := writer.CreateFormFile(bodyName, "filename.txt")
+			if err != nil {
+				return nil, err
+			}
+			io.Copy(part, bytes.NewReader(body.([]byte)))
+			writer.Close()
+			reader = buffer
+		} else {
+			contentType = "application/json"
+			mbytes, err := json.Marshal(body)
+			if err != nil {
+				return nil, err
+			}
+			reader = bytes.NewReader(mbytes)
+		}
+	}
+
+	// create the request object
+	req, err := http.NewRequest(method, triplestoreBaseURL+url, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup configuration!
+	if accept != "" {
+		req.Header.Set("Accept", accept)
+	}
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	req.SetBasicAuth(dis.Config.TriplestoreAdminUser, dis.Config.TriplestoreAdminPassword)
+
+	// and send it
+	return http.DefaultClient.Do(req)
+}
+
+func (dis *Distillery) TriplestoreWaitForConnection() error {
+	return wait.Wait(func() bool {
+		res, err := dis.triplestoreRequest("GET", "/rest/repositories", nil, "", "")
+		if err != nil {
+			return false
+		}
+		defer res.Body.Close()
+		return true
+	}, waitTSInterval, dis.Context())
+}
+
+var errTripleStoreFailedRepository = exit.Error{
+	Message:  "Failed to create repository: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (dis *Distillery) TriplestoreProvision(name, domain, user, password string) error {
+	if err := dis.TriplestoreWaitForConnection(); err != nil {
+		return err
+	}
+
+	// prepare the create repo request
+	createRepo, err := distillery.ReadTemplate(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), map[string]string{
+		"GRAPHDB_REPO":    name,
+		"INSTANCE_DOMAIN": domain,
+	})
+	if err != nil {
+		return err
+	}
+
+	// do the create!
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/repositories", createRepo, "config", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
+		}
+	}
+
+	// create the user and grant them access
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
+			Password: password,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{
+				"ROLE_USER",
+				"READ_REPO_" + name,
+				"WRITE_REPO_" + name,
+			},
+		}, "", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
+		}
+	}
+
+	return nil
+}
+
+// TriplestorePurgeUser deletes the specified user from the triplestore
+func (dis *Distillery) TriplestorePurgeUser(user string) error {
+	res, err := dis.triplestoreRequest("DELETE", "/rest/security/users/"+user, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusNoContent {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+// TriplestorePurgeRepo deletes the specified repo from the triplestore
+func (dis *Distillery) TriplestorePurgeRepo(repo string) error {
+	res, err := dis.triplestoreRequest("DELETE", "/rest/repositories/"+repo, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusOK {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error {
+	logging.LogMessage(io, "Waiting for Triplestore")
+	if err := dis.TriplestoreWaitForConnection(); err != nil {
+		return err
+	}
+
+	logging.LogMessage(io, "Resetting admin user password")
+	{
+		res, err := dis.triplestoreRequest("PUT", "/rest/security/users/"+dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{
+			Password: dis.Config.TriplestoreAdminPassword,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{"ROLE_ADMIN"},
+		}, "", "")
+		if err != nil {
+			return errTriplestoreBootstrap.WithMessageF(err)
+		}
+		defer res.Body.Close()
+
+		switch res.StatusCode {
+		case http.StatusOK:
+			// we set the password => requests are unauthorized
+			// so we still need to enable security (see below!)
+		case http.StatusUnauthorized:
+			// a password is needed => security is already enabled.
+			// the password may or may not work, but that's a problem for later
+			logging.LogMessage(io, "Security is already enabled")
+			return nil
+		default:
+			return errTriplestoreBootstrap.WithMessageF("Unable to set administrative password")
+		}
+	}
+
+	logging.LogMessage(io, "Enabling Triplestore security")
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/security", true, "", "")
+		if err != nil {
+			return errTriplestoreBootstrap.WithMessageF(err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return errTriplestoreBootstrap.WithMessageF("Unable to enable security")
+		}
+
+		return nil
+	}
+}

env/stack_web.go

@@ -0,0 +1,19 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) WebStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "web",
+		},
+
+		EnvFileContext: map[string]string{
+			"DEFAULT_HOST": dis.Config.DefaultDomain,
+		},
+	})
+}
+
+func (dis *Distillery) WebStackPath() string {
+	return dis.WebStack().Dir
+}

go.mod

@@ -0,0 +1,23 @@
+module github.com/FAU-CDI/wisski-distillery
+
+go 1.18
+
+require (
+	github.com/Showmax/go-fqdn v1.0.0
+	github.com/alessio/shellescape v1.4.1
+	github.com/feiin/sqlstring v0.3.0
+	github.com/pkg/errors v0.9.1
+	github.com/tkw1536/goprogram v0.0.9
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
+	gorm.io/driver/mysql v1.3.6
+	gorm.io/gorm v1.23.8
+)
+
+require (
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect
+	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
+)

go.sum

@@ -0,0 +1,30 @@
+github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM=
+github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko=
+github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
+github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/feiin/sqlstring v0.3.0 h1:iyPEFijI2BxpY2M+AuhIvdNManzXa2OwGzuPaEMLUgo=
+github.com/feiin/sqlstring v0.3.0/go.mod h1:xpZTjVUw1nD3hMgF9SMRdPiooKSikLf4PS5j2NTn3RI=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/tkw1536/goprogram v0.0.9 h1:y5bAWbiVRc47TjvpVDmyMtp5CgJXz1ultLOq+v9tfsA=
+github.com/tkw1536/goprogram v0.0.9/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4=
+golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
+golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8=
+golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
+golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM=
+gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
+gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
+gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=

internal/bookkeeping/bookkeeping.go

@@ -0,0 +1,68 @@
+// Package bookkeeping implements reading and writing from the bookkeeping table
+package bookkeeping
+
+import (
+	"database/sql/driver"
+	"errors"
+	"time"
+)
+
+// Instance is a WissKI Instance inside the bookkeeping table.
+// It does not represent a running instance; it does not perform any validation.
+type Instance struct {
+	// NOTE: Modifying this struct requires a database migration.
+	// This should nnever be done unless you know what you're doing.
+
+	// Primary key for the instance
+	Pk uint `gorm:"column:pk;primaryKey"`
+
+	// time the instance was created
+	Created time.Time `gorm:"column:created;autoCreateTime"`
+
+	// slug of the system
+	Slug string `gorm:"column:slug;not null;unique"`
+
+	// email address of the system owner (if any)
+	OwnerEmail string `gorm:"column:owner_email;type:varchar(320)"`
+
+	// should we automatically enable updates for the system?
+	AutoBlindUpdateEnabled SQLBit1 `gorm:"column:auto_blind_update_enabled;default:1"`
+
+	// The filesystem path the system can be found under
+	FilesystemBase string `gorm:"column:filesystem_base;not null"`
+
+	// SQL Database credentials for the system
+	SqlDatabase string `gorm:"column:sql_database;not null"`
+	SqlUser     string `gorm:"column:sql_user;not null"`
+	SqlPassword string `gorm:"column:sql_password;not null"`
+
+	// GraphDB Repository
+	GraphDBRepository string `gorm:"column:graphdb_repository;not null"`
+	GraphDBUser       string `gorm:"column:graphdb_user;not null"`
+	GraphDBPassword   string `gorm:"column:graphdb_password;not null"`
+}
+
+func (i Instance) IsBlindUpdateEnabled() bool {
+	return bool(i.AutoBlindUpdateEnabled)
+}
+
+// SQLBit1 implements a boolean as a BIT(1)
+type SQLBit1 bool
+
+func (sb SQLBit1) Value() (driver.Value, error) {
+	if sb {
+		return []byte{1}, nil
+	} else {
+		return []byte{0}, nil
+	}
+}
+
+var errBadBool = errors.New("SQLBit1: Database does not contain Bit(1)")
+
+func (sb *SQLBit1) Scan(src interface{}) error {
+	if bytes, ok := src.([]byte); ok && len(bytes) == 1 {
+		*sb = bytes[0] == 1
+		return nil
+	}
+	return errBadBool
+}

internal/config/config.go

@@ -0,0 +1,153 @@
+// Package config implements reading and validating a WissKIDistillery configuration file.
+package config
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// Config represents the configuration of a distillery instance
+type Config struct {
+	// Several docker-compose files are created to manage global services and the system itself.
+	// On top of this all real-system space will be created under this directory.
+	DeployRoot string `env:"DEPLOY_ROOT" default:"/var/www/deploy" validator:"is_valid_abspath"`
+
+	// Each created Drupal Instance corresponds to a single domain name.
+	// These domain names should either be a complete domain name or a sub-domain of a default domain.
+	// This setting configures the default domain-name to create subdomains of.
+	DefaultDomain string `env:"DEFAULT_DOMAIN" default:"localhost.kwarc.info" validator:"is_valid_domain"`
+
+	// By default, the default domain redirects to the distillery repository.
+	// If you want to change this, set an alternate domain name here.
+	SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"" validator:"is_valid_https_url"`
+
+	// By default, only the 'self' domain above is caught.
+	// To catch additional domains, add them here (comma seperated)
+	SelfExtraDomains []string `env:"SELF_EXTRA_DOMAINS" default:"" validator:"is_valid_domains"`
+
+	// You can override individual URLS in the homepage
+	// Do this by adding URLs (without trailing '/'s) into a JSON file
+	SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" validator:"is_valid_file"`
+
+	// The system can support setting up certificate(s) automatically.
+	// It can be enabled by setting an email for certbot certificates.
+	// This email address can be configured here.
+	CertbotEmail string `env:"CERTBOT_EMAIL" default:"" validator:"is_valid_email"`
+
+	// Maximum age for backup
+	MaxBackupAge int `env:"MAX_BACKUP_AGE" default:"" validator:"is_valid_number"`
+
+	// Each Drupal instance requires a corresponding system user, database users and databases.
+	// These are also set by the appropriate domain name.
+	// To differentiate them from other users of the system, these names can be prefixed.
+	// The prefix to use can be configured here.
+	// When changing these please consider that no system user may exist that has the same name as a mysql user.
+	// This is a MariaDB restriction.
+	MysqlUserPrefix     string `env:"MYSQL_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
+	MysqlDatabasePrefix string `env:"MYSQL_DATABASE_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
+	GraphDBUserPrefix   string `env:"GRAPHDB_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
+	GraphDBRepoPrefix   string `env:"GRAPHDB_REPO_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
+
+	// In addition to the filesystem the WissKI distillery requires a single SQL table.
+	// It uses this database to store a list of installed things.
+	DistilleryBookkeepingDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" validator:"is_valid_slug"`
+	DistilleryBookkeepingTable    string `env:"DISTILLERY_BOOKKEEPING_TABLE" default:"distillery" validator:"is_valid_slug"`
+
+	// Various components use password-based-authentication.
+	// These passwords are generated automatically.
+	// This variable can be used to determine their length.
+	PasswordLength int `env:"PASSWORD_LENGTH" default:"64" validator:"is_valid_number"`
+
+	// A file to be used for global authorized_keys for the ssh server.
+	GlobalAuthorizedKeysFile string `env:"GLOBAL_AUTHORIZED_KEYS_FILE" default:"/distillery/authorized_keys" validator:"is_valid_file"`
+
+	// admin credentials for graphdb
+	TriplestoreAdminUser     string `env:"GRAPHDB_ADMIN_USER" default:"admin" validator:"is_nonempty"`
+	TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD" default:"" validator:"is_nonempty"`
+
+	// admin credentials for the Mysql database
+	MysqlAdminUser     string `env:"MYSQL_ADMIN_USER" default:"admin" validator:"is_nonempty"`
+	MysqlAdminPassword string `env:"MYSQL_ADMIN_PASSWORD" default:"admin" validator:"is_nonempty"`
+}
+
+func (config Config) String() string {
+	values := &strings.Builder{}
+
+	vConfig := reflect.ValueOf(config)
+	tConfig := vConfig.Type()
+
+	// iterate over the types
+	numValues := tConfig.NumField()
+	for i := 0; i < numValues; i++ {
+		tField := tConfig.Field(i)
+		vField := vConfig.FieldByName(tField.Name)
+
+		fmt.Fprintf(values, "%s=%v\n", tField.Tag.Get("env"), vField.Interface())
+	}
+
+	return values.String()
+}
+
+func (config *Config) Unmarshal(src io.Reader) error {
+	// read all the values!
+	values, err := ReadAll(src)
+	if err != nil {
+		return err
+	}
+
+	vConfig := reflect.ValueOf(config).Elem()
+	tConfig := vConfig.Type()
+
+	// iterate over the types
+	numValues := tConfig.NumField()
+	for i := 0; i < numValues; i++ {
+		tField := tConfig.Field(i)
+		vField := vConfig.FieldByName(tField.Name)
+
+		env := tField.Tag.Get("env")
+		dflt := tField.Tag.Get("default")
+		validator := tField.Tag.Get("validator")
+
+		// read the value with a default
+		value, ok := values[env]
+		if !ok || value == "" {
+			if dflt == "" {
+				continue
+			}
+			value = dflt
+		}
+
+		// use the validator
+		vFunc, ok := knownValidators[validator]
+		if vFunc == nil || !ok {
+			return errors.Errorf("Unable to read %q refers to unknown validator %s", env, validator)
+		}
+
+		// get the parsed value
+		checked, err := vFunc(value)
+		if err != nil {
+			return errors.Wrapf(err, "Unable to read %q: Validator %s", env, validator)
+		}
+
+		// set the value of the field
+		var errSet interface{}
+		func() {
+			defer func() {
+				errSet = recover()
+			}()
+			vField.Set(reflect.ValueOf(checked))
+		}()
+
+		// capture any error
+		if errSet != nil {
+			return errors.Errorf("Unable to parse %q: validator %s returned %q", tField.Name, validator, errSet)
+		}
+	}
+
+	return nil
+}

internal/config/file.go

@@ -0,0 +1,76 @@
+package config
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// Scanner scans an io.Reader for a source file
+type Scanner struct {
+	src *bufio.Scanner
+
+	key   string
+	value string
+}
+
+func NewScanner(r io.Reader) *Scanner {
+	return &Scanner{
+		src: bufio.NewScanner(r),
+	}
+}
+
+// Scanner advances the scanner to the next variable
+func (scanner *Scanner) Scan() bool {
+	for scanner.src.Scan() {
+		// check that we don't have an empty or comment only line
+		tokens := strings.TrimSpace(scanner.src.Text())
+		if len(tokens) == 0 || tokens[0] == '#' || strings.HasPrefix(tokens, "//") {
+			continue
+		}
+
+		// check that we have a 'key=value' pair
+		values := strings.SplitN(tokens, "=", 2)
+		if len(values) != 2 {
+			continue
+		}
+
+		// got a key = value
+		scanner.key = strings.TrimSpace(values[0])
+		scanner.value = strings.TrimSpace(values[1])
+		return true
+	}
+	scanner.key = ""
+	scanner.value = ""
+	return false
+}
+
+// Data reads the current value from the scanner.
+// When Scan() has not been called, or returned false, returns two empty strings.
+func (scanner Scanner) Data() (key, value string) {
+	return scanner.key, scanner.value
+}
+
+// Error returns an error (if any)
+func (scanner Scanner) Error() error {
+	return scanner.src.Err()
+}
+
+// ReadAll reads all key-value pairs from r.
+// If a key occurs more than once, a later occurance overwrites a previous one.
+func ReadAll(r io.Reader) (values map[string]string, err error) {
+	scanner := NewScanner(r)
+
+	// read and store all values
+	values = make(map[string]string)
+	for scanner.Scan() {
+		key, value := scanner.Data()
+		values[key] = value
+	}
+
+	// check if there was an error!
+	if err := scanner.Error(); err != nil {
+		return nil, err
+	}
+	return values, nil
+}

internal/config/validators.go

@@ -0,0 +1,105 @@
+package config
+
+import (
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/pkg/errors"
+)
+
+// Validator reads from the configuration file
+type Validator func(s string) (interface{}, error)
+
+var knownValidators map[string]Validator = map[string]Validator{
+	"is_valid_abspath":   IsValidAbspath,
+	"is_valid_domain":    IsValidDomain,
+	"is_valid_domains":   IsValidDomains,
+	"is_valid_number":    IsValidNumber,
+	"is_valid_https_url": IsValidHttpsURL,
+	"is_valid_slug":      IsValidSlug,
+	"is_valid_file":      IsValidFile,
+	"is_valid_email":     IsValidEmail,
+	"is_nonempty":        IsNonEmpty,
+}
+
+func IsValidAbspath(s string) (interface{}, error) {
+	if !fsx.IsDirectory(s) {
+		return nil, errors.Errorf("%q does not exist or is not a directory", s)
+	}
+	return s, nil
+}
+
+func IsValidFile(s string) (interface{}, error) {
+	if !fsx.IsFile(s) {
+		return nil, errors.Errorf("%q does not exist or is not a regular file", s)
+	}
+	return s, nil
+}
+
+func IsNonEmpty(s string) (interface{}, error) {
+	if s == "" {
+		return nil, errors.New("value is empty")
+	}
+	return s, nil
+}
+
+var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+func IsValidDomain(s string) (interface{}, error) {
+	if !regexpDomain.MatchString(s) {
+		return nil, errors.Errorf("%q is not a valid domain", s)
+	}
+	return s, nil
+}
+func IsValidDomains(s string) (interface{}, error) {
+	if len(s) == 0 {
+		return []string{}, nil
+	}
+	domains := strings.Split(s, ",")
+	for _, d := range domains {
+		if !regexpDomain.MatchString(d) {
+			return nil, errors.Errorf("%q is not a valid domain", d)
+		}
+	}
+	return domains, nil
+}
+
+func IsValidNumber(s string) (interface{}, error) {
+	value, err := strconv.ParseInt(s, 10, 64)
+	return int(value), err
+}
+
+func IsValidHttpsURL(s string) (interface{}, error) {
+	url, err := url.Parse(s)
+	if err != nil {
+		return nil, errors.Wrapf(err, "%q is not a valid URL", s)
+	}
+	if url.Scheme != "https" {
+		return nil, errors.Errorf("%q is not a valid https URL (%q)", s, url.Scheme)
+	}
+	return url, nil
+}
+
+var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+func IsValidEmail(s string) (interface{}, error) {
+	if s == "" { // no email provided
+		return "", nil
+	}
+	if !regexpEmail.MatchString(s) {
+		return nil, errors.Errorf("%q is not a valid email", s)
+	}
+	return s, nil
+}
+
+var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+func IsValidSlug(s string) (interface{}, error) {
+	if !regexpSlug.MatchString(s) {
+		return nil, errors.Errorf("%q is not a valid slug", s)
+	}
+	return s, nil
+}

internal/docs.go

@@ -0,0 +1,4 @@
+// Package internal contains various utility functions.
+//
+// These are not subject to version guarantees and may be changed
+package internal

internal/execx/compose.go

@@ -0,0 +1,11 @@
+package execx
+
+import (
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Compose runs a docker-compose command in a specific directory, with the provided arguments and streams.
+// It then waits for the process to exit, and returns the exit code.
+func Compose(io stream.IOStream, workdir string, args ...string) int {
+	return Exec(io, workdir, "docker", append([]string{"compose"}, args...)...)
+}

internal/execx/exec.go

@@ -0,0 +1,46 @@
+// Package execx defines extensions to the "os/exec" package
+package execx
+
+import (
+	"os/exec"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// ExecCommandError is returned by Exec when a command could not be executed.
+// This typically hints that the executable cannot be found, but may have other causes.
+const ExecCommandError = 127
+
+// Exec executes a system command with the specified input/output streams, working directory, and arguments.
+//
+// If the command executes, it's exit code will be returned.
+// If the command can not be executed, returns [ExecCommandError].
+func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
+	// setup the command
+	cmd := exec.Command(exe, argv...)
+	cmd.Dir = workdir
+	cmd.Stdin = io.Stdin
+	cmd.Stdout = io.Stdout
+	cmd.Stderr = io.Stderr
+
+	// run it
+	err := cmd.Run()
+
+	// non-zero exit
+	if err, ok := err.(*exec.ExitError); ok {
+		return err.ExitCode()
+	}
+
+	// unknown error
+	if err != nil {
+		return ExecCommandError
+	}
+
+	// everything is fine!
+	return 0
+}
+
+// MustExec is like Exec, except that it returns true if the command exited successfully, and else false.
+func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool {
+	return Exec(io, workdir, exe, argv...) == 0
+}

internal/fsx/copy.go

@@ -0,0 +1,41 @@
+package fsx
+
+import (
+	"errors"
+	"io"
+	"os"
+)
+
+var ErrCopySameFile = errors.New("src and dst must be different files")
+
+// CopyFile copies a file from src to dst.
+// When dst and src are the same file, returns ErrCopySameFile.
+func CopyFile(dst, src string) error {
+	if src == dst {
+		return ErrCopySameFile
+	}
+
+	// open the source
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	// stat it to get the mode!
+	srcStat, err := srcFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	// open or create the destination
+	dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode())
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	// and do the copy!
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}

internal/fsx/touch.go

@@ -0,0 +1,25 @@
+package fsx
+
+import (
+	"os"
+	"time"
+)
+
+// Touch touches a file
+func Touch(path string) error {
+	_, err := os.Stat(path)
+	switch {
+	case os.IsNotExist(err):
+		f, err := os.Create(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		return nil
+	case err != nil:
+		return err
+	default:
+		now := time.Now().Local()
+		return os.Chtimes(path, now, now)
+	}
+}

internal/fsx/type.go

@@ -0,0 +1,13 @@
+package fsx
+
+import "os"
+
+func IsDirectory(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.Mode().IsDir()
+}
+
+func IsFile(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.Mode().IsRegular()
+}

internal/hostname/hostname.go

@@ -0,0 +1,32 @@
+// Package hostname provides hostname
+package hostname
+
+import (
+	"os"
+
+	"github.com/Showmax/go-fqdn"
+)
+
+// FQDN attempts to return the fully qualified domain name of the host system.
+// If an error occurs, may fall back to the empty string.
+func FQDN() string {
+
+	// try the hostname function
+	{
+		fqdn, err := fqdn.FqdnHostname()
+		if err == nil {
+			return fqdn
+		}
+	}
+
+	// fallback to os hostname
+	{
+		hostname, err := os.Hostname()
+		if err == nil {
+			return hostname
+		}
+	}
+
+	// use the empty string
+	return ""
+}

internal/legal/legal.go

@@ -0,0 +1,4 @@
+// Package legal contains legal notices.
+package legal
+
+//go:generate gogenlicense -m

internal/logging/level.go

@@ -0,0 +1,59 @@
+package logging
+
+import (
+	"sync"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+var logLevelMutex sync.Mutex
+var logLevelMap = make(map[uintptr]int)
+
+func getIndent(io stream.IOStream) int {
+	logLevelMutex.Lock()
+	defer logLevelMutex.Unlock()
+
+	id, ok := logID(io)
+	if !ok {
+		return 0
+	}
+
+	return logLevelMap[id]
+}
+
+func incIndent(io stream.IOStream) int {
+	logLevelMutex.Lock()
+	defer logLevelMutex.Unlock()
+
+	id, ok := logID(io)
+	if !ok { // if we don't have an id, then inc statically returns 1
+		return 1
+	}
+
+	logLevelMap[id]++
+	return logLevelMap[id]
+}
+
+func decIndent(io stream.IOStream) int {
+	logLevelMutex.Lock()
+	defer logLevelMutex.Unlock()
+	id, ok := logID(io)
+
+	if !ok { // if we don't have an id, then dec statically returns 0
+		return 0
+	}
+
+	logLevelMap[id]--
+	if logLevelMap[id] < 0 {
+		panic("DecLogIdent: decrease below 0")
+	}
+	return logLevelMap[id]
+}
+
+func logID(io stream.IOStream) (uintptr, bool) {
+	file, ok := io.Stdin.(interface{ Fd() uintptr })
+	if !ok {
+		return 0, false
+	}
+	return file.Fd(), true
+}

internal/logging/log.go

@@ -0,0 +1,30 @@
+package logging
+
+import (
+	"strings"
+
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// LogOperation logs a message that is displayed to the user, and then increases the log indent level.
+func LogOperation(operation func() error, io stream.IOStream, format string, args ...interface{}) error {
+	logOperation(io, getIndent(io), format, args...)
+	incIndent(io)
+	defer decIndent(io)
+
+	return operation()
+}
+
+// LogMessage logs a message that is displayed to the user
+func LogMessage(io stream.IOStream, format string, args ...interface{}) (int, error) {
+	return logOperation(io, getIndent(io), format, args...)
+}
+
+func logOperation(io stream.IOStream, indent int, format string, args ...interface{}) (int, error) {
+	message := "\033[1m" + strings.Repeat(" ", indent+1) + "=> " + format + "\033[0m\n"
+	if !io.StdinIsATerminal() {
+		message = " => " + format
+	}
+
+	return io.Printf(message, args...)
+}

internal/password/password.go

@@ -0,0 +1,41 @@
+// Package password allows generating random passwords
+package password
+
+import (
+	"crypto/rand"
+	"math/big"
+	"strings"
+)
+
+// NOTE(twiesing): A bunch of scripts cannot properly handle the extra characters in the password.
+// For now it is disabled, but it should be re-enabled later.
+const PasswordCharSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // + "!@#$%&*"
+const PasswordCharCount = len(PasswordCharSet)
+
+// Password returns a randomly generated password with the provided length.
+// [rand.Reader] is used as the source of randomness.
+func Password(length int) (string, error) {
+	if length < 0 {
+		panic("length < 0")
+	}
+
+	var password strings.Builder
+	password.Grow(length)
+
+	for i := 0; i < length; i++ {
+
+		// grab a random index!
+		index, err := rand.Int(rand.Reader, big.NewInt(int64(PasswordCharCount)))
+		if err != nil {
+			return "", err
+		}
+
+		// and use that index!
+		if err := password.WriteByte(PasswordCharSet[int(index.Int64())]); err != nil {
+			return "", err
+		}
+	}
+
+	// return the password!
+	return password.String(), nil
+}

internal/sqle/sqle.go

@@ -0,0 +1,10 @@
+package sqle
+
+import (
+	"github.com/feiin/sqlstring"
+)
+
+// Format formats the provided query with the given parameters.
+func Format(query string, params ...interface{}) string {
+	return sqlstring.Format(query, params...)
+}

internal/stack/installable.go

@@ -0,0 +1,108 @@
+package stack
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Installable represents a Stack that can be automatically installed from a set of resources
+// See the Install() method.
+type Installable struct {
+	Stack
+
+	ContextResource string // Path to the resource containing 'docker compose' context
+
+	EnvFileResource string            // Path to the resource containing dynamically generated env file
+	EnvFileContext  map[string]string // Context of variables to replace in the env file
+
+	CopyContextFiles []string // Files to copy from the installation context
+
+	TouchFiles []string // Files to 'touch', i.e. ensure that exist
+
+	MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
+	MakeDirs     []string    // directories to ensure that exist
+}
+
+// InstallationContext is a context to install data in
+type InstallationContext map[string]string
+
+// Install installs or updates this stack into the directory specified by stack.Stack().
+//
+// Installation is non-interactive, but will provide debugging output onto io.
+// InstallationContext
+func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
+	// setup the base files
+	if err := distillery.InstallResource(
+		is.Dir,
+		is.ContextResource,
+		func(dst, src string) {
+			io.Printf("[install] %s\n", dst)
+		},
+	); err != nil {
+		return err
+	}
+
+	// configure .env
+	envDest := filepath.Join(is.Dir, ".env")
+	if is.EnvFileResource != "" && is.EnvFileContext != nil {
+		io.Printf("[config]  %s\n", envDest)
+		if err := distillery.InstallTemplate(
+			envDest,
+			is.EnvFileResource,
+			is.EnvFileContext,
+		); err != nil {
+			return err
+		}
+	}
+
+	// make sure that certain files exist
+	for _, name := range is.MakeDirs {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[make]    %s\n", dst)
+		if is.MakeDirsPerm == fs.FileMode(0) {
+			is.MakeDirsPerm = fs.ModeDir
+		}
+		if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
+			return err
+		}
+	}
+
+	// copy files from the context!
+	for _, name := range is.CopyContextFiles {
+		// find the source!
+		src, ok := context[name]
+		if !ok {
+			return errors.Errorf("Missing file from context: %s", src)
+		}
+
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		// copy over file from context
+		io.Printf("[copy]    %s (from %s)\n", dst, src)
+		if err := fsx.CopyFile(dst, src); err != nil {
+			return errors.Wrapf(err, "Unable to copy file %s", src)
+		}
+	}
+
+	// make sure that certain files exist
+	for _, name := range is.TouchFiles {
+		// find the destination!
+		dst := filepath.Join(is.Dir, name)
+
+		io.Printf("[touch]   %s\n", dst)
+		if err := fsx.Touch(dst); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/stack/stack.go

@@ -0,0 +1,110 @@
+// Package stack implements a docker compose stack
+package stack
+
+import (
+	"errors"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/execx"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Stack represents a 'docker compose' stack living in a specific directory
+//
+// NOTE(twiesing): In the current implementation this requires a 'docker' executable on the system.
+// This executable must be capable of the 'docker compose' command.
+// In the future the idea is to replace this with a native docker compose client.
+type Stack struct {
+	Name string // Name of this stack, TODO: Do we need this?
+	Dir  string // Directory of this stack
+}
+
+var errStackUpdatePull = errors.New("Stack.Update: Pull returned non-zero exit code")
+var errStackUpdateBuild = errors.New("Stack.Update: Build returned non-zero exit code")
+
+// Update pulls, builds, and then optionally starts this stack.
+// This does not have a direct 'docker compose' shell equivalent.
+//
+// See also Up.
+func (ds Stack) Update(io stream.IOStream, start bool) error {
+	if ds.compose(io, "pull") != 0 {
+		return errStackUpdatePull
+	}
+	if ds.compose(io, "build", "--pull") != 0 {
+		return errStackUpdateBuild
+	}
+	if start {
+		return ds.Up(io)
+	}
+	return nil
+}
+
+var errStackUp = errors.New("Stack.Up: Up returned non-zero exit code")
+
+// Up creates and starts the containers in this Stack.
+// It is equivalent to 'docker compose up -d' on the shell.
+func (ds Stack) Up(io stream.IOStream) error {
+	if ds.compose(io, "up", "-d") != 0 {
+		return errStackUp
+	}
+	return nil
+}
+
+// Exec executes an executable in the provided running service.
+// It is equivalent to 'docker compose exec $service $executable $args...'.
+//
+// It returns the exit code of the process.
+func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...string) int {
+	compose := []string{"exec"}
+	if io.StdinIsATerminal() {
+		compose = append(compose, "-ti")
+	}
+	compose = append(compose, executable)
+	compose = append(compose, args...)
+	return ds.compose(io, compose...)
+}
+
+// Run executes the provided service with the given executable.
+// It is equivalent to 'docker compose run [--rm] $service $executable $args...'.
+//
+// It returns the exit code of the process.
+func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string, args ...string) int {
+	compose := []string{"run"}
+	if autoRemove {
+		compose = append(compose, "--rm")
+	}
+	if !io.StdinIsATerminal() {
+		compose = append(compose, "-T")
+	}
+	compose = append(compose, command)
+	compose = append(compose, args...)
+	return ds.compose(io, compose...)
+}
+
+var errStackRestart = errors.New("Stack.Restart: Restart returned non-zero exit code")
+
+// Restart restarts all containers in this Stack.
+// It is equivalent to 'docker compose restart' on the shell.
+func (ds Stack) Restart(io stream.IOStream) error {
+	if ds.compose(io, "restart") != 0 {
+		return errStackRestart
+	}
+	return nil
+}
+
+var errStackDown = errors.New("Stack.Down: Down returned non-zero exit code")
+
+// Down stops and removes all containers in this Stack.
+// It is equivalent to 'docker compose down -v' on the shell.
+func (ds Stack) Down(io stream.IOStream) error {
+	if ds.compose(io, "down", "-v") != 0 {
+		return errStackDown
+	}
+	return nil
+}
+
+// Compose executes a 'docker compose' command on this stack.
+// TODO: This should be removed and replaced by an internal call directly to libcompose.
+func (ds Stack) compose(io stream.IOStream, args ...string) int {
+	// TODO: can we migrate to a built-in version of this?
+	return execx.Compose(io, ds.Dir, args...)
+}

internal/wait/wait.go

@@ -0,0 +1,31 @@
+package wait
+
+import (
+	"context"
+	"time"
+)
+
+// Wait repeatedly invokes f, until it returns true or the context is closed.
+// The invocation interval is determined by interval.
+func Wait(f func() bool, interval time.Duration, context context.Context) error {
+	// create a new timer
+	timer := time.NewTimer(interval)
+	if !timer.Stop() {
+		<-timer.C
+	}
+	defer timer.Stop()
+
+	for {
+		if f() {
+			return nil
+		}
+
+		// reset the timer, and wait for it again!
+		timer.Reset(interval)
+		select {
+		case <-timer.C:
+		case <-context.Done():
+			return context.Err()
+		}
+	}
+}

license.go

@@ -0,0 +1,6 @@
+package wisski_distillery
+
+import _ "embed"
+
+//go:embed LICENSE
+var License string

program.go

@@ -0,0 +1,44 @@
+package wisski_distillery
+
+import (
+	"os/user"
+
+	"github.com/FAU-CDI/wisski-distillery/env"
+	"github.com/tkw1536/goprogram"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// these define the ggman-specific program types
+// none of these are strictly needed, they're just around for convenience
+type wdcliEnv = *env.Distillery
+type wdcliParameters = env.Params
+type wdcliRequirements = env.Requirements
+type wdCliFlags = struct{}
+
+type Program = goprogram.Program[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements]
+type Command = goprogram.Command[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements]
+type Context = goprogram.Context[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements]
+type Arguments = goprogram.Arguments[wdCliFlags]
+type Description = goprogram.Description[wdCliFlags, wdcliRequirements]
+
+// an error when nor arguments are provided.
+var errUserIsNotRoot = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "This command has to be executed as root. The current user is not root.",
+}
+
+func NewProgram() Program {
+	return Program{
+		BeforeCommand: func(context Context, command Command) error {
+			usr, err := user.Current()
+			if err != nil || usr.Uid != "0" || usr.Gid != "0" { // make sure that we are root!
+				return errUserIsNotRoot
+			}
+			return nil
+		},
+
+		NewEnvironment: func(params wdcliParameters, context Context) (e wdcliEnv, err error) {
+			return env.NewDistillery(params, context.Description.Requirements)
+		},
+	}
+}