Do a large chunk of the move to go

This commit moves a huge chunk of the code to go. The TODO.md document
indicates what is left to be done.

env/dirs.go

@@ -0,0 +1,23 @@
+package env
+
+import "path/filepath"
+
+func (dis Distillery) BackupDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "backups")
+}
+
+func (dis Distillery) RuntimeDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "runtime")
+}
+
+func (dis Distillery) RuntimeUtilsDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "runtime", "utils")
+}
+
+func (dis Distillery) InprogressBackupPath() string {
+	return filepath.Join(dis.BackupDir(), "inprogress")
+}
+
+func (dis Distillery) FinalBackupPath() string {
+	return filepath.Join(dis.BackupDir(), "final")
+}

env/distillery.go

@@ -0,0 +1,77 @@
+package env
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Distillery represents a running instance for the distillery
+type Distillery struct {
+	Config *config.Config
+}
+
+func (dis Distillery) HTTPSEnabled() bool {
+	return dis.Config.CertbotEmail != ""
+}
+
+// Returns the default virtual host
+func (dis Distillery) DefaultVirtualHost() string {
+	VIRTUAL_HOST := dis.Config.DefaultDomain
+	if len(dis.Config.SelfExtraDomains) > 0 {
+		VIRTUAL_HOST += "," + strings.Join(dis.Config.SelfExtraDomains, ",")
+	}
+	return VIRTUAL_HOST
+}
+
+func (dis Distillery) DefaultLetsencryptHost() string {
+	if !dis.HTTPSEnabled() {
+		return ""
+	}
+	return dis.DefaultVirtualHost()
+}
+
+// Context returns a new Context belonging to this distillery
+func (dis Distillery) Context() context.Context {
+	return context.Background()
+}
+
+var errNoConfigFile = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Configuration File does not exist",
+}
+
+var errOpenConfig = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "error loading configuration file: %s",
+}
+
+// NewDistillery creates a new distillery object from a set of parameters and requirements
+func NewDistillery(params Params, req Requirements) (env *Distillery, err error) {
+	env = &Distillery{}
+
+	// if we don't need to load the config, there is nothing to do
+	if !req.NeedsConfig {
+		return
+	}
+
+	// if there is no no config file, return
+	cfg := params.ConfigFilePath()
+	if cfg == "" {
+		return nil, errNoConfigFile
+	}
+
+	f, err := os.Open(params.ConfigFilePath())
+	if err != nil {
+		return nil, errOpenConfig.WithMessageF(err)
+	}
+	defer f.Close()
+
+	// unmarshal the config
+	env.Config = &config.Config{}
+	err = env.Config.Unmarshal(f)
+	return
+}

env/instances.go

@@ -0,0 +1,359 @@
+package env
+
+import (
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/alessio/shellescape"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+var errNoBookkeeping = exit.Error{
+	Message:  "instance %q does not exist in bookkeeping table",
+	ExitCode: exit.ExitGeneric,
+}
+
+var ErrInstanceNotFound = exit.Error{
+	Message:  "instance not found",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Instance returns the instance of the WissKI Distillery with the provided slug
+func (dis *Distillery) Instance(slug string) (i Instance, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return i, err
+	}
+
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return i, err
+	}
+
+	// find the instance by slug
+	query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
+	switch {
+	case query.Error != nil:
+		return i, errSQL.WithMessageF(query.Error)
+	case query.RowsAffected == 0:
+		return i, ErrInstanceNotFound
+	default:
+		i.dis = dis
+		return i, nil
+	}
+}
+
+// HasInstance checks if the provided instance exists in the bookeeping table
+func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return false, err
+	}
+
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return false, err
+	}
+
+	query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok)
+	if query.Error != nil {
+		return false, errSQL.WithMessageF(query.Error)
+	}
+	return
+}
+
+// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances.
+func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) {
+	if len(slugs) == 0 {
+		return dis.AllInstances()
+	}
+	return dis.InstancesWith(slugs...)
+}
+
+// AllInstances returns all instances of the WissKI Distillery in consistent order.
+//
+// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order.
+func (dis *Distillery) AllInstances() ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table
+	})
+}
+
+// InstancesWith returns all instances where the slug is in the provided list of names.
+// The returned instances are reordered in a consistent order.
+func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table.Where("slug IN ?", slugs)
+	})
+}
+
+// findInstances finds instance objects based on a query in the bookkeeping table
+func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
+	if err := dis.SQLWaitForConnection(); err != nil {
+		return nil, err
+	}
+
+	// open the bookkeeping table
+	table, err := dis.sqlBkTable(false)
+	if err != nil {
+		return nil, err
+	}
+
+	// prepare a query
+	find := table
+	if order {
+		find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false})
+	}
+	if query != nil {
+		find = query(find)
+	}
+
+	// fetch bookkeeping instances
+	var bks []bookkeeping.Instance
+	find = find.Find(&bks)
+	if find.Error != nil {
+		return nil, errSQL.WithMessageF(find.Error)
+	}
+
+	// make proper instances
+	instances = make([]Instance, len(bks))
+	for i, bk := range bks {
+		instances[i].Instance = bk
+		instances[i].dis = dis
+	}
+
+	return instances, nil
+}
+
+// Instance represents a bookkeeping instance
+type Instance struct {
+	bookkeeping.Instance
+
+	// Credentials for the drupal instance
+	DrupalUsername string
+	DrupalPassword string
+
+	dis *Distillery
+}
+
+// Update updates the bookkeeping table with this instance.
+func (instance *Instance) Update() error {
+	db, err := instance.dis.sqlBkTable(false)
+	if err != nil {
+		return err
+	}
+
+	// it has never been created => we need to create it in the database
+	if instance.Instance.Created.IsZero() {
+		return db.Create(&instance.Instance).Error
+	}
+
+	// Update based on the primary key!
+	return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error
+}
+
+// Delete deletes this instance from the bookkeeping table
+func (instance *Instance) Delete() error {
+	db, err := instance.dis.sqlBkTable(false)
+	if err != nil {
+		return err
+	}
+
+	// doesn't exist => nothing to delete
+	if instance.Instance.Created.IsZero() {
+		return nil
+	}
+
+	// delete it directly
+	return db.Delete(&instance.Instance).Error
+}
+
+// Shell executes a shell command inside the
+func (instance Instance) Shell(io stream.IOStream, argv ...string) int {
+	return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...)
+}
+
+// Domain returns the full domain name of this instance
+func (instance Instance) Domain() string {
+	return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain)
+}
+
+// IfHttps returns value if the distillery has https enabled, the empty string otherwise
+// TODO: Fix this to be in a proper place
+func (dis *Distillery) IfHttps(value string) string {
+	if !dis.HTTPSEnabled() {
+		return ""
+	}
+	return value
+}
+
+// URL returns the public URL of this instance
+func (instance Instance) URL() *url.URL {
+	// setup domain and path
+	url := &url.URL{
+		Host: instance.Domain(),
+		Path: "/",
+	}
+
+	// use http or https scheme depending on if the distillery has it enabled
+	if instance.dis.HTTPSEnabled() {
+		url.Scheme = "https"
+	} else {
+		url.Scheme = "http"
+	}
+
+	return url
+}
+
+// Stack represents a stack representing this instance
+func (instance Instance) Stack() stack.Installable {
+	return stack.Installable{
+		Stack: stack.Stack{
+			Name: "barrel",
+			Dir:  instance.FilesystemBase,
+		},
+		ContextResource: filepath.Join("resources", "compose", "barrel"),
+
+		EnvFileResource: filepath.Join("resources", "templates", "docker-env", "barrel"),
+		EnvFileContext: map[string]string{
+			"REAL_PATH": instance.FilesystemBase,
+
+			"SLUG":         instance.Slug,
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail),
+
+			"UTILS_DIR":                   instance.dis.RuntimeUtilsDir(),
+			"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
+		},
+
+		CopyContextFiles: nil,
+
+		TouchFiles: []string{
+			"authorized_keys",
+		},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs:     []string{"data", ".composer"},
+	}
+}
+
+func (instance Instance) ReserveStack() stack.Installable {
+	return stack.Installable{
+		Stack: stack.Stack{
+			Name: "reserve",
+			Dir:  instance.FilesystemBase,
+		},
+		ContextResource: filepath.Join("resources", "compose", "reserve"),
+
+		EnvFileResource: filepath.Join("resources", "templates", "docker-env", "reserve"),
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail),
+		},
+	}
+}
+
+// Provision provisions an instance, assuming that the required databases already exist.
+func (instance Instance) Provision(io stream.IOStream) error {
+
+	// create the basic st!
+	st := instance.Stack()
+	if err := st.Install(io, stack.InstallationContext{}); err != nil {
+		return err
+	}
+
+	// Pull and build the stack!
+	if err := st.Update(io, false); err != nil {
+		return err
+	}
+
+	provisionParams := []string{
+		instance.Domain(),
+
+		instance.SqlDatabase,
+		instance.SqlUser,
+		instance.SqlPassword,
+
+		instance.GraphDBRepository,
+		instance.GraphDBUser,
+		instance.GraphDBPassword,
+
+		instance.DrupalUsername,
+		instance.DrupalPassword,
+
+		"", // TODO: DrupalVersion
+		"", // TODO: WissKIVersion
+	}
+
+	// escape the parameter
+	for i, param := range provisionParams {
+		provisionParams[i] = shellescape.Quote(param)
+	}
+
+	// figure out the provision script
+	// TODO: Move the provision script into the control plane!
+	provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
+
+	if st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) != 0 {
+		return errors.New("Unable to run provision script")
+	}
+
+	return nil
+}
+
+func (instance *Instance) NoPrefix() bool {
+	return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip"))
+}
+
+var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
+
+// PrefixConfig returns the prefix config belonging to this instance.
+func (instance *Instance) PrefixConfig() (config string, err error) {
+	// if the user requested to skip the prefix, then don't do anything with it!
+	if instance.NoPrefix() {
+		return "", nil
+	}
+
+	var builder strings.Builder
+
+	// domain
+	builder.WriteString(instance.URL().String() + ":")
+	builder.WriteString("\n")
+
+	// default prefixes
+	wu := stream.NewIOStream(&builder, nil, nil, 0)
+	if instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") != 0 {
+		return "", errPrefixExecFailed
+	}
+
+	// custom prefixes
+	prefixPath := filepath.Join(instance.FilesystemBase, "prefixes")
+	if fsx.IsFile(prefixPath) {
+		prefix, err := os.Open(prefixPath)
+		if err != nil {
+			return "", err
+		}
+		defer prefix.Close()
+		if _, err := io.Copy(&builder, prefix); err != nil {
+			return "", err
+		}
+		builder.WriteString("\n")
+	}
+
+	// and done!
+	return builder.String(), nil
+}

env/instances_provision.go

@@ -0,0 +1,96 @@
+package env
+
+import (
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/password"
+	"github.com/pkg/errors"
+)
+
+func (dis *Distillery) InstancesDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "instances")
+}
+
+func (dis *Distillery) InstanceDir(slug string) string {
+	return filepath.Join(dis.InstancesDir(), slug)
+}
+
+func (dis *Distillery) InstanceSQL(slug string) (database, user string) {
+	database = dis.Config.MysqlDatabasePrefix + slug
+	user = dis.Config.MysqlUserPrefix + slug
+	return
+}
+
+func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) {
+	repo = dis.Config.GraphDBRepoPrefix + slug
+	user = dis.Config.GraphDBUserPrefix + slug
+	return
+}
+
+// Password returns a new password
+func (dis *Distillery) NewPassword() (value string, err error) {
+	return password.Password(dis.Config.PasswordLength)
+}
+
+var errInvalidSlug = errors.New("Not a valid slug")
+
+// NewInstance fills the struct for a new distillery instance.
+// It validates that slug is a valid name for an instance.
+//
+// It does not perform any checks if the instance already exists, or does the creation in the database.
+func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
+
+	// make sure that the slug is valid!
+	if _, err := config.IsValidSlug(slug); err != nil {
+		return i, errInvalidSlug
+	}
+
+	// generate sql data
+	sqlPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	sqlDB, sqlUser := dis.InstanceSQL(slug)
+
+	// generate ts data
+	tsPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	tsRepo, tsUser := dis.InstanceGraphDB(slug)
+
+	// generate drupal data
+	drPassword, err := dis.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	drUser := "admin"
+
+	// make the instance object!
+	instance := bookkeeping.Instance{
+		Slug: slug,
+
+		OwnerEmail:             "",
+		AutoBlindUpdateEnabled: true,
+
+		FilesystemBase: dis.InstanceDir(slug),
+
+		SqlDatabase: sqlDB,
+		SqlUser:     sqlUser,
+		SqlPassword: sqlPassword,
+
+		GraphDBRepository: tsRepo,
+		GraphDBUser:       tsUser,
+		GraphDBPassword:   tsPassword,
+	}
+
+	i.DrupalUsername = drUser
+	i.DrupalPassword = drPassword
+
+	// store the instance in the object and return it!
+	i.Instance = instance
+	i.dis = dis
+	return i, nil
+}

env/params.go

@@ -0,0 +1,92 @@
+package env
+
+import (
+	"errors"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Params are parameters used for initialization of the environment
+type Params struct {
+	BaseDirectory string
+}
+
+// ConfigFilePath returns the path to the configuration file
+func (params Params) ConfigFilePath() string {
+	if params.BaseDirectory == "" {
+		return ""
+	}
+	return filepath.Join(params.BaseDirectory, ".env")
+}
+
+var errUnableToLoadParams = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Unable to configure wdcli environment: %s",
+}
+
+const BaseDirectoryDefault = "/var/www/deploy"
+
+// ParamsFromEnv creates a new set of parameters from the environment.
+// There is no guarantee that the parameters are valid.
+func ParamsFromEnv() (params Params, err error) {
+	// try to read the base directory
+	value, err := ReadBaseDirectory()
+	switch {
+	case os.IsNotExist(err):
+		params.BaseDirectory = BaseDirectoryDefault
+	case err == nil:
+		params.BaseDirectory = value
+	default:
+		return params, errUnableToLoadParams.WithMessageF(err)
+	}
+
+	return params, nil
+}
+
+var baseConfigFile = ".wdcli"
+
+// ReadBaseDirectory reads the base directory from the environment, or an empty string
+func ReadBaseDirectory() (value string, err error) {
+	// find the current user
+	usr, err := user.Current()
+	if err != nil {
+		return "", err
+	}
+
+	// read the base config file!
+	contents, err := os.ReadFile(filepath.Join(usr.HomeDir, baseConfigFile))
+	if err != nil {
+		return "", err
+	}
+
+	// and trim the spaces!
+	value = strings.TrimSpace(string(contents))
+
+	// check that it is actually set!
+	if len(value) == 0 {
+		return "", errors.New("ReadBaseDirectory: Directory is empty")
+	}
+
+	// and return it!
+	return value, nil
+}
+
+// WriteBaseDirectory writes the base directory to the environment, or returns an error
+func WriteBaseDirectory(dir string) error {
+	// find the current user
+	usr, err := user.Current()
+	if err != nil {
+		return err
+	}
+
+	// read the base config file!
+	return os.WriteFile(
+		filepath.Join(usr.HomeDir, baseConfigFile),
+		[]byte(dir),
+		os.ModePerm,
+	)
+}

env/requirements.go

@@ -0,0 +1,24 @@
+package env
+
+import (
+	"github.com/tkw1536/goprogram"
+	"github.com/tkw1536/goprogram/meta"
+)
+
+type Requirements struct {
+	NeedsConfig bool
+}
+
+// AllowsFlag checks if the provided flag may be passed to fullfill this requirement
+// By default it is used only for help page generation, and may be inaccurate.
+func (r Requirements) AllowsFlag(flag meta.Flag) bool {
+	return true
+}
+
+// Validate validates if this requirement is fullfilled for the provided global flags.
+// It should return either nil, or an error of type exit.Error.
+//
+// Validate does not take into account AllowsOption, see ValidateAllowedOptions.
+func (r Requirements) Validate(arguments goprogram.Arguments[struct{}]) error {
+	return nil
+}

env/stack.go

@@ -0,0 +1,30 @@
+package env
+
+import (
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+// Stacks returns the Stacks of this distillery
+func (dis *Distillery) Stacks() []stack.Installable {
+	// TODO: Do we want to cache these stacks?
+	return []stack.Installable{
+		dis.WebStack(),
+		dis.SelfStack(),
+		dis.ResolverStack(),
+		dis.SSHStack(),
+		dis.TriplestoreStack(),
+		dis.SQLStack(),
+	}
+}
+
+// asCoreStack treats the provided stack as a core component of this distillery.
+func (dis *Distillery) asCoreStack(stack stack.Installable) stack.Installable {
+	stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", stack.Name)
+
+	stack.ContextResource = filepath.Join("resources", "compose", stack.Name)
+	stack.EnvFileResource = filepath.Join("resources", "templates", "docker-env", stack.Name)
+
+	return stack
+}

env/stack_resolver.go

@@ -0,0 +1,39 @@
+package env
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+const ResolverPrefixFile = "prefix.cfg"
+
+func (dis *Distillery) ResolverStack() stack.Installable {
+	stack := dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "resolver",
+		},
+
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST":      dis.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  dis.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
+			"PREFIX_FILE":       "", // set below!
+			"DEFAULT_DOMAIN":    dis.Config.DefaultDomain,
+			"LEGACY_DOMAIN":     strings.Join(dis.Config.SelfExtraDomains, ","),
+		},
+
+		TouchFiles: []string{ResolverPrefixFile},
+	})
+	stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, ResolverPrefixFile)
+	return stack
+}
+
+func (dis *Distillery) ResolverStackPath() string {
+	return dis.ResolverStack().Dir
+}
+
+func (dis Distillery) ResolverPrefixConfig() string {
+	return filepath.Join(dis.ResolverStackPath(), ResolverPrefixFile)
+}

env/stack_self.go

@@ -0,0 +1,28 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) SelfStack() stack.Installable {
+	TARGET := "https://github.com/FAU-CDI/wisski-distillery"
+	if dis.Config.SelfRedirect != nil {
+		TARGET = dis.Config.SelfRedirect.String()
+	}
+
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "self",
+		},
+
+		EnvFileContext: map[string]string{
+			"VIRTUAL_HOST":      dis.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  dis.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
+			"TARGET":            TARGET,
+			"OVERRIDES_FILE":    dis.Config.SelfOverridesFile,
+		},
+	})
+}
+
+func (dis *Distillery) SelfStackPath() string {
+	return dis.SelfStack().Dir
+}

env/stack_sql.go

@@ -0,0 +1,221 @@
+package env
+
+import (
+	"fmt"
+	"io/fs"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/sqle"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+// SQLStack returns the docker stack that handles the sql database.
+func (dis *Distillery) SQLStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "sql",
+		},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			"data",
+		},
+	})
+}
+
+// SQLStackPath returns the path the SQLStack() lives at.
+func (dis *Distillery) SQLStackPath() string {
+	return dis.SQLStack().Dir
+}
+
+// sqlOpen opens a new sql connection to the provided database using the administrative credentials
+func (env Distillery) sqlOpen(database string, config *gorm.Config) (*gorm.DB, error) {
+	sql := mysql.Config{
+		DSN:               fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", env.Config.MysqlAdminUser, env.Config.MysqlAdminPassword, "127.0.0.1:3306", database),
+		DefaultStringSize: 256,
+	}
+
+	db, err := gorm.Open(mysql.New(sql), config)
+	if err != nil {
+		return db, err
+	}
+
+	gdb, err := db.DB()
+	if err != nil {
+		return db, err
+	}
+	gdb.SetMaxIdleConns(0)
+
+	return db, nil
+}
+
+var errSQL = exit.Error{
+	Message:  "error querying sql database: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+// sqlBkTable returns a gorm connection to the bookkeeping database.
+func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) {
+
+	config := &gorm.Config{}
+	if silent {
+		config.Logger = logger.Default.LogMode(logger.Silent)
+	}
+
+	// open the database
+	db, err := dis.sqlOpen(dis.Config.DistilleryBookkeepingDatabase, config)
+	if err != nil {
+		return nil, errSQL.WithMessageF(err)
+	}
+
+	// load the table
+	table := db.Table(dis.Config.DistilleryBookkeepingTable)
+	if table.Error != nil {
+		return nil, errSQL.WithMessageF(err)
+	}
+
+	return table, nil
+}
+
+// SQLShell executes a mysql shell inside the SQLStack.
+func (dis *Distillery) SQLShell(io stream.IOStream, argv ...string) int {
+	return dis.SQLStack().Exec(io, "sql", "mysql", argv...)
+}
+
+var errSQLBootstrap = exit.Error{
+	Message:  "Unable to boostrap SQL: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+const waitSQLInterval = 1 * time.Second
+
+// SQLWaitForShell waits for the sql database to be reachable via a docker-compose shell
+func (dis *Distillery) SQLWaitForShell() error {
+	n := stream.FromNil()
+	return wait.Wait(func() bool {
+		return dis.SQLShell(n, "-e", "show databases;") == 0
+	}, waitSQLInterval, dis.Context())
+}
+
+// SQLWaitForConnection waits for the sql connection to be alive
+func (dis *Distillery) SQLWaitForConnection() error {
+	return wait.Wait(func() bool {
+		_, err := dis.sqlBkTable(true)
+		return err == nil
+	}, waitSQLInterval, dis.Context())
+}
+
+var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
+
+func (dis *Distillery) sqlRaw(query string, args ...interface{}) bool {
+	sql := sqle.Format(query, args...)
+	return dis.SQLShell(stream.FromNil(), "-e", sql) == 0
+}
+
+// SQLProvision provisions a new sql database and user
+func (dis *Distillery) SQLProvision(name, user, password string) error {
+	// wait for the database
+	if err := dis.SQLWaitForShell(); err != nil {
+		return err
+	}
+
+	// it's not a safe database name!
+	if !sqle.IsSafeDatabaseName(name) {
+		return errInvalidDatabaseName
+	}
+
+	// create the database and user!
+	if !dis.sqlRaw("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
+		return errors.New("SQLProvision: Failed to create user")
+	}
+
+	// and done!
+	return nil
+}
+
+var errSQLPurgeUser = exit.Error{
+	Message:  "Unable to delete user",
+	ExitCode: exit.ExitGeneric,
+}
+
+// SQLPurgeUser deletes the specified user from the database
+func (dis *Distillery) SQLPurgeUser(user string) error {
+	if !dis.sqlRaw("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
+		return errSQLPurgeUser
+	}
+
+	return nil
+}
+
+var errSQLPurgeDB = exit.Error{
+	Message:  "Unable to delete database",
+	ExitCode: exit.ExitGeneric,
+}
+
+// SQLPurgeDatabase deletes the specified db from the database
+func (dis *Distillery) SQLPurgeDatabase(db string) error {
+	if !sqle.IsSafeDatabaseName(db) {
+		return errSQLPurgeDB
+	}
+	if !dis.sqlRaw("DROP DATABASE IF EXISTS `" + db + "`") {
+		return errSQLPurgeDB
+	}
+	return nil
+}
+
+// SQLBootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
+func (dis *Distillery) SQLBootstrap(io stream.IOStream) error {
+	if err := dis.SQLWaitForShell(); err != nil {
+		return errSQLBootstrap.WithMessageF(err)
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating administrative user")
+	{
+		username := dis.Config.MysqlAdminUser
+		password := dis.Config.MysqlAdminPassword
+		if !dis.sqlRaw("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
+			return errSQLBootstrap.WithMessageF("Unable to create administrative user")
+		}
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating sql database")
+	{
+		if !sqle.IsSafeDatabaseName(dis.Config.DistilleryBookkeepingDatabase) {
+			return errSQLBootstrap.WithMessageF("Unsafe database name")
+		}
+		createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", dis.Config.DistilleryBookkeepingDatabase)
+		if !dis.sqlRaw(createDBSQL) {
+			return errSQLBootstrap.WithMessageF(createDBSQL)
+		}
+	}
+
+	// wait for the database to come up
+	logging.LogMessage(io, "Waiting for database update to be complete")
+	dis.SQLWaitForConnection()
+
+	// open the database
+	logging.LogMessage(io, "Migrating bookkeeping table")
+	{
+		db, err := dis.sqlBkTable(false)
+		if err != nil {
+			return errSQLBootstrap.WithMessageF(err)
+		}
+
+		if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
+			return errSQLBootstrap.WithMessageF(err)
+		}
+	}
+
+	return nil
+}

env/stack_ssh.go

@@ -0,0 +1,16 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) SSHStack() stack.Installable {
+	// TODO: Ensure that .env is copied if needed
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "sql",
+		},
+	})
+}
+
+func (dis *Distillery) SSHStackPath() string {
+	return dis.SSHStack().Dir
+}

env/stack_triplestore.go

@@ -0,0 +1,263 @@
+package env
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/fs"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (dis *Distillery) TriplestoreStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "triplestore",
+		},
+
+		CopyContextFiles: []string{"graphdb.zip"},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			filepath.Join("data", "data"),
+			filepath.Join("data", "work"),
+			filepath.Join("data", "logs"),
+		},
+	})
+}
+
+func (dis *Distillery) TriplestoreStackPath() string {
+	return dis.TriplestoreStack().Dir
+}
+
+type TriplestoreUserPayload struct {
+	Password           string                     `json:"password"`
+	AppSettings        TriplestoreUserAppSettings `json:"appSettings"`
+	GrantedAuthorities []string                   `json:"grantedAuthorities"`
+}
+type TriplestoreUserAppSettings struct {
+	DefaultInference      bool `json:"DEFAULT_INFERENCE"`
+	DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
+	DefaultSameas         bool `json:"DEFAULT_SAMEAS"`
+	IgnoreSharedQueries   bool `json:"IGNORE_SHARED_QUERIES"`
+	ExecuteCount          bool `json:"EXECUTE_COUNT"`
+}
+
+var errTriplestoreBootstrap = exit.Error{
+	Message:  "Unable to bootstrap Triplestore: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+const triplestoreBaseURL = "http://127.0.0.1:7200"
+const waitTSInterval = 1 * time.Second
+
+// triplestoreCall makes a request to the triplestore.
+//
+// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
+// When bodyName is empty, simply marshal body as application/json
+func (dis *Distillery) triplestoreRequest(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
+	var reader io.Reader
+
+	var contentType string
+
+	// for "PUT" and "POST" we setup a body
+	if method == "PUT" || method == "POST" {
+		if bodyName != "" {
+			buffer := &bytes.Buffer{}
+			writer := multipart.NewWriter(buffer)
+			contentType = writer.FormDataContentType()
+
+			part, err := writer.CreateFormFile(bodyName, "filename.txt")
+			if err != nil {
+				return nil, err
+			}
+			io.Copy(part, bytes.NewReader(body.([]byte)))
+			writer.Close()
+			reader = buffer
+		} else {
+			contentType = "application/json"
+			mbytes, err := json.Marshal(body)
+			if err != nil {
+				return nil, err
+			}
+			reader = bytes.NewReader(mbytes)
+		}
+	}
+
+	// create the request object
+	req, err := http.NewRequest(method, triplestoreBaseURL+url, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup configuration!
+	if accept != "" {
+		req.Header.Set("Accept", accept)
+	}
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	req.SetBasicAuth(dis.Config.TriplestoreAdminUser, dis.Config.TriplestoreAdminPassword)
+
+	// and send it
+	return http.DefaultClient.Do(req)
+}
+
+func (dis *Distillery) TriplestoreWaitForConnection() error {
+	return wait.Wait(func() bool {
+		res, err := dis.triplestoreRequest("GET", "/rest/repositories", nil, "", "")
+		if err != nil {
+			return false
+		}
+		defer res.Body.Close()
+		return true
+	}, waitTSInterval, dis.Context())
+}
+
+var errTripleStoreFailedRepository = exit.Error{
+	Message:  "Failed to create repository: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (dis *Distillery) TriplestoreProvision(name, domain, user, password string) error {
+	if err := dis.TriplestoreWaitForConnection(); err != nil {
+		return err
+	}
+
+	// prepare the create repo request
+	createRepo, err := distillery.ReadTemplate(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), map[string]string{
+		"GRAPHDB_REPO":    name,
+		"INSTANCE_DOMAIN": domain,
+	})
+	if err != nil {
+		return err
+	}
+
+	// do the create!
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/repositories", createRepo, "config", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
+		}
+	}
+
+	// create the user and grant them access
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
+			Password: password,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{
+				"ROLE_USER",
+				"READ_REPO_" + name,
+				"WRITE_REPO_" + name,
+			},
+		}, "", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
+		}
+	}
+
+	return nil
+}
+
+// TriplestorePurgeUser deletes the specified user from the triplestore
+func (dis *Distillery) TriplestorePurgeUser(user string) error {
+	res, err := dis.triplestoreRequest("DELETE", "/rest/security/users/"+user, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusNoContent {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+// TriplestorePurgeRepo deletes the specified repo from the triplestore
+func (dis *Distillery) TriplestorePurgeRepo(repo string) error {
+	res, err := dis.triplestoreRequest("DELETE", "/rest/repositories/"+repo, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusOK {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error {
+	logging.LogMessage(io, "Waiting for Triplestore")
+	if err := dis.TriplestoreWaitForConnection(); err != nil {
+		return err
+	}
+
+	logging.LogMessage(io, "Resetting admin user password")
+	{
+		res, err := dis.triplestoreRequest("PUT", "/rest/security/users/"+dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{
+			Password: dis.Config.TriplestoreAdminPassword,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{"ROLE_ADMIN"},
+		}, "", "")
+		if err != nil {
+			return errTriplestoreBootstrap.WithMessageF(err)
+		}
+		defer res.Body.Close()
+
+		switch res.StatusCode {
+		case http.StatusOK:
+			// we set the password => requests are unauthorized
+			// so we still need to enable security (see below!)
+		case http.StatusUnauthorized:
+			// a password is needed => security is already enabled.
+			// the password may or may not work, but that's a problem for later
+			logging.LogMessage(io, "Security is already enabled")
+			return nil
+		default:
+			return errTriplestoreBootstrap.WithMessageF("Unable to set administrative password")
+		}
+	}
+
+	logging.LogMessage(io, "Enabling Triplestore security")
+	{
+		res, err := dis.triplestoreRequest("POST", "/rest/security", true, "", "")
+		if err != nil {
+			return errTriplestoreBootstrap.WithMessageF(err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return errTriplestoreBootstrap.WithMessageF("Unable to enable security")
+		}
+
+		return nil
+	}
+}

env/stack_web.go

@@ -0,0 +1,19 @@
+package env
+
+import "github.com/FAU-CDI/wisski-distillery/internal/stack"
+
+func (dis *Distillery) WebStack() stack.Installable {
+	return dis.asCoreStack(stack.Installable{
+		Stack: stack.Stack{
+			Name: "web",
+		},
+
+		EnvFileContext: map[string]string{
+			"DEFAULT_HOST": dis.Config.DefaultDomain,
+		},
+	})
+}
+
+func (dis *Distillery) WebStackPath() string {
+	return dis.WebStack().Dir
+}