Move code into new component package

This commit cleans up the resources in the 'embed' package, and instead
moves them into subpackages of a new 'compose' package. This makes sure
that '.env' templates and docker compose contexts are located in the
same location.

component/component.go

@@ -0,0 +1,61 @@
+// Package component holds the main abstraction for components.
+package component
+
+import (
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+// Component represents a logical subsystem of the distillery.
+//
+// By convention these are defined within their corresponding subpackage.
+// This subpackage also contains all required resources.
+// Furthermore, a component is typically instantiated using a call on the ["distillery.Distillery"] struct.
+//
+// Each Component should make use of [ComponentBase] for sane defaults.
+//
+// For example, the web.Web component lives in the web package and can be created like:
+//
+//	var dis Distillery
+//  web := dis.Web()
+type Component interface {
+	// Name returns the name of this component.
+	// It should correspond to the appropriate subpackage.
+	Name() string
+
+	// Path returns the path this component is installed at.
+	// By convention it is /var/www/deploy/core/${Name()}
+	Path() string
+
+	// Stack can be used to gain access to the "docker compose" stack.
+	//
+	// This should internally call
+	Stack() stack.Installable
+
+	// Context returns a new InstallationContext to be used during installation from the command line.
+	// Typically this should just pass through the parent, but might perform other tasks.
+	Context(parent stack.InstallationContext) stack.InstallationContext
+}
+
+// ComponentBase implements base functionality for a component
+type ComponentBase struct {
+	Dir string // Dir is the directory this component lives in
+
+	Config *config.Config // Config is the configuration of the underlying distillery
+}
+
+// Path returns the path to this component
+func (cb ComponentBase) Path() string {
+	return cb.Dir
+}
+
+// Context passes through the parent context
+func (ComponentBase) Context(parent stack.InstallationContext) stack.InstallationContext {
+	return parent
+}
+
+// MakeStack registers the Installable as a stack
+func (cb ComponentBase) MakeStack(stack stack.Installable) stack.Installable {
+	stack.Dir = cb.Dir
+	return stack
+}

component/dis/dis.env

@@ -0,0 +1,9 @@
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
+
+CONFIG_PATH=${CONFIG_PATH}
+DEPLOY_ROOT=${DEPLOY_ROOT}
+GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
+SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}

component/dis/dis.go

@@ -0,0 +1,51 @@
+package dis
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+type Dis struct {
+	component.ComponentBase
+
+	// TODO: SQL Component
+
+	Executable string // path to the current executable
+}
+
+func (dis Dis) Name() string {
+	return "dis"
+}
+
+//go:embed all:stack dis.env
+var resources embed.FS
+
+func (dis Dis) Stack() stack.Installable {
+	return dis.ComponentBase.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+		EnvPath:     "dis.env",
+
+		EnvContext: map[string]string{
+			"VIRTUAL_HOST":      dis.Config.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  dis.Config.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
+
+			"CONFIG_PATH": dis.Config.ConfigPath,
+			"DEPLOY_ROOT": dis.Config.DeployRoot,
+
+			"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
+			"SELF_OVERRIDES_FILE":         dis.Config.SelfOverridesFile,
+		},
+		CopyContextFiles: []string{core.Executable},
+	})
+}
+
+func (dis Dis) Context(parent stack.InstallationContext) stack.InstallationContext {
+	return stack.InstallationContext{
+		core.Executable: dis.Executable,
+	}
+}

component/dis/stack/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.io/library/alpine
+
+COPY wdcli /wdcli
+EXPOSE 8888
+CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","dis_server","--bind","0.0.0.0:8888"]

component/dis/stack/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.7"
+
+services:
+  wdresolve:
+    build: .
+    restart: always
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8888
+      VIRTUAL_PATH: /dis/
+
+      CONFIG_PATH: ${CONFIG_PATH}
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+
+    volumes:
+      - "${CONFIG_PATH}:${CONFIG_PATH}:ro"
+      - "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
+      - "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
+      - "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
+
+networks:
+  default:
+    name: distillery
+    external: true

component/resolver/resolver.env

@@ -0,0 +1,10 @@
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
+
+CONFIG_PATH=${CONFIG_PATH}
+DEPLOY_ROOT=${DEPLOY_ROOT}
+GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
+SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
+RESOLVER_CONFIG=${RESOLVER_CONFIG}

component/resolver/resolver.go

@@ -0,0 +1,110 @@
+package resolver
+
+import (
+	"embed"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/FAU-CDI/wdresolve"
+	"github.com/FAU-CDI/wdresolve/resolvers"
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// TODO: Add a 'self-server' concept!
+
+type Resolver struct {
+	component.ComponentBase
+
+	ConfigName string // the name to the config file
+	Executable string // path to the current executable
+}
+
+func (Resolver) Name() string {
+	return "resolver"
+}
+
+func (resolver Resolver) ConfigPath() string {
+	return filepath.Join(resolver.Dir, resolver.ConfigName)
+}
+
+//go:embed all:stack resolver.env
+var resources embed.FS
+
+func (resolver Resolver) Stack() stack.Installable {
+	return resolver.ComponentBase.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+		EnvPath:     "resolver.env",
+
+		EnvContext: map[string]string{
+			"VIRTUAL_HOST":      resolver.Config.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  resolver.Config.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": resolver.Config.CertbotEmail,
+
+			"CONFIG_PATH": resolver.Config.ConfigPath,
+			"DEPLOY_ROOT": resolver.Config.DeployRoot,
+
+			"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.Config.GlobalAuthorizedKeysFile,
+			"SELF_OVERRIDES_FILE":         resolver.Config.SelfOverridesFile,
+			"RESOLVER_CONFIG":             resolver.ConfigPath(),
+		},
+		TouchFiles:       []string{resolver.ConfigName},
+		CopyContextFiles: []string{core.Executable},
+	})
+}
+
+func (resolver Resolver) Context(parent stack.InstallationContext) stack.InstallationContext {
+	return stack.InstallationContext{
+		core.Executable: resolver.Executable,
+	}
+}
+
+func (resolver Resolver) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
+	p.TrustXForwardedProto = true
+
+	fallback := &resolvers.Regexp{
+		Data: map[string]string{},
+	}
+
+	// handle the default domain name!
+	domainName := resolver.Config.DefaultDomain
+	if domainName != "" {
+		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
+		io.Printf("registering default domain %s\n", domainName)
+	}
+
+	// handle the extra domains!
+	for _, domain := range resolver.Config.SelfExtraDomains {
+		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
+		io.Printf("registering legacy domain %s\n", domain)
+	}
+
+	// open the prefix file
+	prefixFile := resolver.ConfigPath()
+	fs, err := os.Open(prefixFile)
+	io.Println("loading prefixes from ", prefixFile)
+	if err != nil {
+		return p, err
+	}
+	defer fs.Close()
+
+	// read the prefixes
+	// TODO: Do we want to load these without a file?
+	prefixes, err := resolvers.ReadPrefixes(fs)
+	if err != nil {
+		return p, err
+	}
+
+	// and use that as the resolver!
+	p.Resolver = resolvers.InOrder{
+		prefixes,
+		fallback,
+	}
+
+	return p, nil
+}

component/resolver/stack/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.io/library/alpine
+
+COPY wdcli /wdcli
+EXPOSE 8888
+CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]

component/resolver/stack/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3.7"
+
+services:
+  wdresolve:
+    build: .
+    restart: always
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8888
+      VIRTUAL_PATH: /go/
+
+      CONFIG_PATH: ${CONFIG_PATH}
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+
+    volumes:
+      - "${CONFIG_PATH}:${CONFIG_PATH}:ro"
+      - "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
+      - "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
+      - "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
+      - "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
+
+networks:
+  default:
+    name: distillery
+    external: true

component/self/self.env

@@ -0,0 +1,7 @@
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
+
+TARGET=${TARGET}
+OVERRIDES_FILE=${OVERRIDES_FILE}

component/self/self.go

@@ -0,0 +1,43 @@
+package self
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+type Self struct {
+	component.ComponentBase
+}
+
+func (Self) Name() string {
+	return "self"
+}
+
+//go:embed all:stack
+//go:embed self.env
+var resources embed.FS
+
+func (self Self) Stack() stack.Installable {
+	// TODO: Move me into config!
+	TARGET := "https://github.com/FAU-CDI/wisski-distillery"
+	if self.Config.SelfRedirect != nil { // TODO: move to config!
+		TARGET = self.Config.SelfRedirect.String()
+	}
+
+	return self.ComponentBase.MakeStack(stack.Installable{
+		Resources: resources,
+
+		ContextPath: "stack",
+		EnvPath:     "self.env",
+
+		EnvContext: map[string]string{
+			"VIRTUAL_HOST":      self.Config.DefaultVirtualHost(),
+			"LETSENCRYPT_HOST":  self.Config.DefaultLetsencryptHost(),
+			"LETSENCRYPT_EMAIL": self.Config.CertbotEmail,
+			"TARGET":            TARGET,
+			"OVERRIDES_FILE":    self.Config.SelfOverridesFile,
+		},
+	})
+}

component/self/stack/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.7"
+
+services:
+  tr:
+    image: ghcr.io/tkw1536/tr:latest
+    restart: always
+    volumes:
+      - "${OVERRIDES_FILE}:/overrides.json:ro"
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8080
+      VIRTUAL_PATH: /
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+
+      # the overrides file
+      OVERRIDES: /overrides.json
+
+      # where to redirect to
+      TARGET: ${TARGET}
+
+networks:
+  default:
+    name: distillery
+    external: true

component/sql/database.go

@@ -0,0 +1,217 @@
+package sql
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/sqle"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/tkw1536/goprogram/stream"
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+// sqlOpen opens a new sql connection to the provided database using the administrative credentials
+func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
+	cfg := mysql.Config{
+		DSN:               fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
+		DefaultStringSize: 256,
+	}
+
+	db, err := gorm.Open(mysql.New(cfg), config)
+	if err != nil {
+		return db, err
+	}
+
+	gdb, err := db.DB()
+	if err != nil {
+		return db, err
+	}
+	gdb.SetMaxIdleConns(0)
+
+	return db, nil
+}
+
+// OpenBookkeeping opens a connection to the bookkeeping database
+func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
+
+	config := &gorm.Config{}
+	if silent {
+		config.Logger = logger.Default.LogMode(logger.Silent)
+	}
+
+	// open the database
+	db, err := sql.openDatabase(sql.Config.DistilleryBookkeepingDatabase, config)
+	if err != nil {
+		return nil, err
+	}
+
+	// load the table
+	table := db.Table(sql.Config.DistilleryBookkeepingTable)
+	if table.Error != nil {
+		return nil, err
+	}
+
+	return table, nil
+}
+
+var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
+
+// Backup makes a backup of the sql database into dest.
+func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
+	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
+
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}
+
+// BackupAll makes a backup of all sql databases
+func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
+	io = stream.NewIOStream(dest, io.Stderr, nil, 0)
+
+	code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}
+
+// OpenShell executes a mysql shell command
+func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
+	return sql.Stack().Exec(io, "sql", "mysql", argv...)
+}
+
+// WaitShell waits for the sql database to be reachable via a docker-compose shell
+func (sql SQL) WaitShell() error {
+	n := stream.FromNil()
+	return wait.Wait(func() bool {
+		code, err := sql.OpenShell(n, "-e", "show databases;")
+		return err == nil && code == 0
+	}, sql.PollInterval, sql.PollContext)
+}
+
+// Wait waits for a connection to the bookkeeping table to suceed
+func (sql SQL) Wait() error {
+	return wait.Wait(func() bool {
+		_, err := sql.OpenBookkeeping(true)
+		return err == nil
+	}, sql.PollInterval, sql.PollContext)
+}
+
+var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
+
+func (sql SQL) Query(query string, args ...interface{}) bool {
+	raw := sqle.Format(query, args...)
+	code, err := sql.OpenShell(stream.FromNil(), "-e", raw)
+	return err == nil && code == 0
+}
+
+// SQLProvision provisions a new sql database and user
+func (sql SQL) Provision(name, user, password string) error {
+	// wait for the database
+	if err := sql.WaitShell(); err != nil {
+		return err
+	}
+
+	// it's not a safe database name!
+	if !sqle.IsSafeDatabaseName(name) {
+		return errInvalidDatabaseName
+	}
+
+	// create the database and user!
+	if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
+		return errors.New("SQLProvision: Failed to create user")
+	}
+
+	// and done!
+	return nil
+}
+
+var errSQLPurgeUser = errors.New("unable to delete user")
+
+// SQLPurgeUser deletes the specified user from the database
+func (sql SQL) PurgeUser(user string) error {
+	if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
+		return errSQLPurgeUser
+	}
+
+	return nil
+}
+
+var errSQLPurgeDB = errors.New("unable to drop database")
+
+// SQLPurgeDatabase deletes the specified db from the database
+func (sql SQL) PurgeDatabase(db string) error {
+	if !sqle.IsSafeDatabaseName(db) {
+		return errSQLPurgeDB
+	}
+	if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") {
+		return errSQLPurgeDB
+	}
+	return nil
+}
+
+var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
+var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe name")
+var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
+
+// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
+func (sql SQL) Bootstrap(io stream.IOStream) error {
+	if err := sql.WaitShell(); err != nil {
+		return err
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating administrative user")
+	{
+		username := sql.Config.MysqlAdminUser
+		password := sql.Config.MysqlAdminPassword
+		if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
+			return errSQLUnableToCreateUser
+		}
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating sql database")
+	{
+		if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) {
+			return errSQLUnsafeDatabaseName
+		}
+		createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase)
+		if !sql.Query(createDBSQL) {
+			return errSQLUnableToCreate
+		}
+	}
+
+	// wait for the database to come up
+	logging.LogMessage(io, "Waiting for database update to be complete")
+	sql.Wait()
+
+	// open the database
+	logging.LogMessage(io, "Migrating bookkeeping table")
+	{
+		db, err := sql.OpenBookkeeping(false)
+		if err != nil {
+			return fmt.Errorf("unable to access bookkeeping table: %s", err)
+		}
+
+		if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
+			return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
+		}
+	}
+
+	return nil
+}

component/sql/sql.go

@@ -0,0 +1,39 @@
+package sql
+
+import (
+	"context"
+	"embed"
+	"io/fs"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+type SQL struct {
+	component.ComponentBase
+
+	ServerURL string // upstream server url
+
+	PollContext  context.Context // context to abort polling with
+	PollInterval time.Duration   // duration to wait for during wait
+}
+
+func (SQL) Name() string {
+	return "sql"
+}
+
+//go:embed all:stack
+var resources embed.FS
+
+func (ssh SQL) Stack() stack.Installable {
+	return ssh.ComponentBase.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			"data",
+		},
+	})
+}

component/sql/stack/docker-compose.yml

@@ -0,0 +1,35 @@
+version: "3.7"
+
+services:
+    sql:
+        image: mariadb
+        volumes:
+            - "./data/:/var/lib/mysql"
+        ports:
+            - 127.0.0.1:3306:3306
+        environment:
+            # This combination of environment variables will configure a passwordless root user
+            # that can only connect to the container from 'localhost'.
+            # This means we can only connect using 'docker-compose exec sql mysql -C '...' '. 
+            - "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
+            - "MYSQL_ROOT_HOST=localhost"
+        restart: always
+    phpmyadmin:
+        image: phpmyadmin/phpmyadmin
+        environment:
+            - "PMA_HOST=sql"
+            - "HIDE_PHP_VERSION=true"
+            - "UPLOAD_LIMIT=100M"
+        # phpmyadmin running on localhost:8080 so that we can easily access the system graphically. 
+        # By default no admin account is created, so initial shell access to make one is needed. 
+        ports:
+            - 127.0.0.1:8080:80
+        depends_on:
+            - sql
+        restart: always
+
+
+networks:
+    default:
+        name: distillery
+        external: true

component/ssh/ssh.go

@@ -0,0 +1,26 @@
+package ssh
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+type SSH struct {
+	component.ComponentBase
+}
+
+func (SSH) Name() string {
+	return "ssh"
+}
+
+//go:embed all:stack
+var resources embed.FS
+
+func (ssh SSH) Stack() stack.Installable {
+	return ssh.ComponentBase.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+	})
+}

component/ssh/stack/docker-compose.yml

@@ -0,0 +1,17 @@
+version: "3.7"
+
+services:
+    ssh:
+        image: ghcr.io/tkw1536/dockersshd:latest
+        command: -hostkey /keys/hostkey -shell /user_shell.sh -keylabel eu.wiss-ki.barrel.authfile -userlabel eu.wiss-ki.barrel.slug -L triplestore:7200 -L phpmyadmin:80 -L sql:3306
+        ports:
+            - "2222:2222"
+        volumes:
+            - './data/keys:/keys'
+            - '/var/run/docker.sock:/var/run/docker.sock:ro'
+        restart: always
+
+networks:
+    default:
+        name: distillery
+        external: true

component/triplestore/database.go

@@ -0,0 +1,315 @@
+package triplestore
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/embed"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
+	"github.com/FAU-CDI/wisski-distillery/internal/logging"
+	"github.com/FAU-CDI/wisski-distillery/internal/unpack"
+	"github.com/FAU-CDI/wisski-distillery/internal/wait"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+type TriplestoreUserPayload struct {
+	Password           string                     `json:"password"`
+	AppSettings        TriplestoreUserAppSettings `json:"appSettings"`
+	GrantedAuthorities []string                   `json:"grantedAuthorities"`
+}
+type TriplestoreUserAppSettings struct {
+	DefaultInference      bool `json:"DEFAULT_INFERENCE"`
+	DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
+	DefaultSameas         bool `json:"DEFAULT_SAMEAS"`
+	IgnoreSharedQueries   bool `json:"IGNORE_SHARED_QUERIES"`
+	ExecuteCount          bool `json:"EXECUTE_COUNT"`
+}
+
+// OpenRaw makes an http request to the triplestore api.
+//
+// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
+// When bodyName is empty, simply marshal body as application/json
+func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
+	var reader io.Reader
+
+	var contentType string
+
+	// for "PUT" and "POST" we setup a body
+	if method == "PUT" || method == "POST" {
+		if bodyName != "" {
+			buffer := &bytes.Buffer{}
+			writer := multipart.NewWriter(buffer)
+			contentType = writer.FormDataContentType()
+
+			part, err := writer.CreateFormFile(bodyName, "filename.txt")
+			if err != nil {
+				return nil, err
+			}
+			io.Copy(part, bytes.NewReader(body.([]byte)))
+			writer.Close()
+			reader = buffer
+		} else {
+			contentType = "application/json"
+			mbytes, err := json.Marshal(body)
+			if err != nil {
+				return nil, err
+			}
+			reader = bytes.NewReader(mbytes)
+		}
+	}
+
+	// create the request object
+	req, err := http.NewRequest(method, ts.BaseURL+url, reader)
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup configuration!
+	if accept != "" {
+		req.Header.Set("Accept", accept)
+	}
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
+
+	// and send it
+	return http.DefaultClient.Do(req)
+}
+
+// Wait waits for the connection to the Triplestore to succeed.
+// This is achieved using a polling strategy.
+func (ts Triplestore) Wait() error {
+	return wait.Wait(func() bool {
+		res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
+		if err != nil {
+			return false
+		}
+		defer res.Body.Close()
+		return true
+	}, ts.PollInterval, ts.PollContext)
+}
+
+var errTripleStoreFailedRepository = exit.Error{
+	Message:  "Failed to create repository: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+func (ts Triplestore) Provision(name, domain, user, password string) error {
+	if err := ts.Wait(); err != nil {
+		return err
+	}
+
+	// prepare the create repo request
+	// TODO: Move this into a seperate file
+	createRepo, _, err := unpack.UnpackTemplate(
+		map[string]string{
+			"GRAPHDB_REPO":    name,
+			"INSTANCE_DOMAIN": domain,
+		},
+		fsx.OpenFS(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), embed.ResourceEmbed),
+	)
+	if err != nil {
+		return err
+	}
+
+	// do the create!
+	{
+		res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo, "config", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
+		}
+	}
+
+	// create the user and grant them access
+	{
+		res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
+			Password: password,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{
+				"ROLE_USER",
+				"READ_REPO_" + name,
+				"WRITE_REPO_" + name,
+			},
+		}, "", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
+		}
+	}
+
+	return nil
+}
+
+// TriplestorePurgeUser deletes the specified user from the triplestore
+func (ts Triplestore) PurgeUser(user string) error {
+	res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusNoContent {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+// TriplestorePurgeRepo deletes the specified repo from the triplestore
+func (ts Triplestore) PurgeRepo(repo string) error {
+	res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
+	if err != nil {
+		return err
+	}
+	if res.StatusCode != http.StatusOK {
+		return errors.Errorf("Delete returned code %d", res.StatusCode)
+	}
+	return nil
+}
+
+var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
+
+// TriplestoreBackup backs up the repository named repo into the writer dst.
+func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
+	res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
+	if err != nil {
+		return 0, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return 0, errTSBackupWrongStatusCode
+	}
+	defer res.Body.Close()
+	return io.Copy(dst, res.Body)
+}
+
+type Repository struct {
+	ID         string `json:"id"`
+	Title      string `json:"title"`
+	URI        string `json:"uri"`
+	Type       string `json:"type"`
+	SesameType string `json:"sesameType"`
+	Location   string `json:"location"`
+	Readable   bool   `json:"readable"`
+	Writable   bool   `json:"writable"`
+	Local      bool   `json:"local"`
+}
+
+func (ts Triplestore) listRepositories() (repos []Repository, err error) {
+	res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	err = json.NewDecoder(res.Body).Decode(&repos)
+	return
+}
+
+// TriplestoreBackup backs up every graphdb instance into dst
+func (ts Triplestore) BackupAll(dst string) error {
+	// list all the repositories
+	repos, err := ts.listRepositories()
+	if err != nil {
+		return err
+	}
+
+	// create the base directory
+	if err := os.Mkdir(dst, fs.ModeDir); err != nil {
+		return err
+	}
+
+	// iterate over all the repositories
+	for _, repo := range repos {
+		if rErr := (func(repo Repository) error {
+			name := filepath.Join(dst, repo.ID+".nq")
+
+			dest, err := os.Create(name)
+			if err != nil {
+				return err
+			}
+			defer dest.Close()
+
+			_, err = ts.Backup(dest, repo.ID)
+			return err
+		}(repo)); err == nil && rErr != nil {
+			err = rErr
+		}
+	}
+	return err
+}
+
+var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
+
+func (ts Triplestore) Bootstrap(io stream.IOStream) error {
+	logging.LogMessage(io, "Waiting for Triplestore")
+	if err := ts.Wait(); err != nil {
+		return err
+	}
+
+	logging.LogMessage(io, "Resetting admin user password")
+	{
+		res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
+			Password: ts.Config.TriplestoreAdminPassword,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{"ROLE_ADMIN"},
+		}, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+		defer res.Body.Close()
+
+		switch res.StatusCode {
+		case http.StatusOK:
+			// we set the password => requests are unauthorized
+			// so we still need to enable security (see below!)
+		case http.StatusUnauthorized:
+			// a password is needed => security is already enabled.
+			// the password may or may not work, but that's a problem for later
+			logging.LogMessage(io, "Security is already enabled")
+			return nil
+		default:
+			return fmt.Errorf("failed to create triplestore user: %s", err)
+		}
+	}
+
+	logging.LogMessage(io, "Enabling Triplestore security")
+	{
+		res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
+		if err != nil {
+			return fmt.Errorf("failed to enable triplestore security: %s", err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return errTriplestoreFailedSecurity
+		}
+
+		return nil
+	}
+}

component/triplestore/stack/.dockerignore

@@ -0,0 +1,3 @@
+*
+!*.zip
+!entrypoint.sh

component/triplestore/stack/Dockerfile

@@ -0,0 +1,64 @@
+# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container. 
+# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
+# but has been modified for performance and security. 
+
+# This image is intended to be built like:
+#   docker build --build-arg graphdb_src=graphdb.zip .
+
+# We first make a base image to base further builds on. 
+# We don't use alpine here, as that uses significantly slower musl instead of glibc. 
+FROM adoptopenjdk/openjdk11:debian-slim as base
+
+# Create a user called graphdb
+RUN useradd -ms /bin/bash graphdb
+
+# make a base images, to add the sources to. 
+FROM base as sources
+
+# install unzip
+RUN apt-get update && apt-get install -y unzip
+
+# add the source file (by default graphdb.zip) to the image
+ARG src=graphdb.zip
+ADD ${src} /graphdb.zip
+
+# unpack it into a temporary directory
+RUN unzip "$src" -d "/unpack/"
+
+# Move it into /opt/graphdb, and chown it to graphdb
+RUN mv "/unpack"/* /opt/graphdb
+RUN chown -R graphdb:graphdb /opt/graphdb
+
+# finally make an image that will run
+FROM base as final
+
+# add the entrypoint script
+ADD entrypoint.sh /entrypoint.sh
+
+# copy over the sources
+COPY --from=sources /opt/graphdb /opt/graphdb
+
+# set environment variables for graphdb_home and path
+ENV GRAPHDB_HOME=/opt/graphdb
+ENV PATH=$GRAPHDB_HOME/bin:$PATH
+
+# Workaround for CVE-2021-44228 
+# (not sure if we are vulnerable, but just because)
+ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
+
+# expose a port
+EXPOSE 7200
+
+# setup a healthcheck, that checks if the server is up. 
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
+
+# Add volumes for data, work and logs as these might be accessible from the outside. 
+# To add your own configuration, manually mount a config file into /opt/graphdb/work
+VOLUME /opt/graphdb/data
+VOLUME /opt/graphdb/work
+VOLUME /opt/graphdb/logs
+
+# setup command and entrypoint
+CMD ["-Dgraphdb.home=/opt/graphdb"]
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

component/triplestore/stack/docker-compose.yml

@@ -0,0 +1,22 @@
+version: "3.7"
+
+services:
+    triplestore:
+        build: .
+        ports:
+            - "127.0.0.1:7200:7200"
+        volumes:
+            - './data/data:/opt/graphdb/data'
+            - './data/work:/opt/graphdb/work'
+            - './data/logs:/opt/graphdb/logs'
+        command: "\"-Dgraphdb.home=/opt/graphdb -Ddefault.min.distinct.threshold=2G\""
+        # Use 1GB of heap space
+        environment:
+            GDB_HEAP_SIZE: 16G
+
+        restart: always
+
+networks:
+    default:
+        name: distillery
+        external: true

component/triplestore/stack/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Because we want to run graphdb as a limited user
+# we need to make sure that the volumes are writable. 
+# Because of that, we 'chown'
+
+chown graphdb:graphdb /opt/graphdb/data
+chown graphdb:graphdb /opt/graphdb/work
+chown graphdb:graphdb /opt/graphdb/logs
+
+# switch to the graphdb user, and run graphdb
+su graphdb -c "/opt/graphdb/bin/graphdb $@"

component/triplestore/triplestore.go

@@ -0,0 +1,44 @@
+package triplestore
+
+import (
+	"context"
+	"embed"
+	"io/fs"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+type Triplestore struct {
+	component.ComponentBase
+
+	BaseURL string // upstream server url
+
+	PollContext  context.Context // context to abort polling with
+	PollInterval time.Duration   // duration to wait for during wait
+}
+
+func (Triplestore) Name() string {
+	return "triplestore"
+}
+
+//go:embed all:stack
+var resources embed.FS
+
+func (ts Triplestore) Stack() stack.Installable {
+	return ts.ComponentBase.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+
+		CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs: []string{
+			filepath.Join("data", "data"),
+			filepath.Join("data", "work"),
+			filepath.Join("data", "logs"),
+		},
+	})
+}

component/web/stack/docker-compose.yml

@@ -0,0 +1,52 @@
+version: "3.7"
+
+services:
+  nginx-proxy:
+    image: ghcr.io/nginx-proxy/nginx-proxy:alpine
+    environment:
+      - DEFAULT_HOST=${DEFAULT_HOST}
+      - HTTPS_METHOD=${HTTPS_METHOD}
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "vhost:/etc/nginx/vhost.d"
+      - "./global.conf:/etc/nginx/conf.d/global.conf:ro"
+      - "./proxy.conf:/etc/nginx/proxy.conf:ro"
+      - "htpasswd:/etc/nginx/htpasswd"
+      - "html:/usr/share/nginx/html"
+      - "/var/run/docker.sock:/tmp/docker.sock:ro"
+      - "certs:/etc/nginx/certs"
+    labels:
+      com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: true
+    restart: always
+    networks:
+      - default
+
+  letsencrypt-nginx-proxy-companion:
+    image: docker.io/nginxproxy/acme-companion:latest
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "htpasswd:/etc/nginx/htpasswd"
+      - "vhost:/etc/nginx/vhost.d"
+      - "html:/usr/share/nginx/html"
+      - "/var/run/docker.sock:/tmp/docker.sock:ro"
+      - "certs:/etc/nginx/certs"
+      - "acme:/etc/acme.sh"
+    restart: always
+    networks:
+      - default
+    depends_on:
+      - nginx-proxy
+
+volumes:
+  acme:
+  vhost:
+  html:
+  certs:
+  htpasswd:
+
+networks:
+  default:
+    name: distillery
+    external: true

component/web/stack/global.conf

@@ -0,0 +1,4 @@
+# Nginx Configuration File
+# These should match with distillery/resources/compose/barrel/conf/wisski.ini.
+ 
+client_max_body_size 1000m;

component/web/stack/proxy.conf

@@ -0,0 +1,19 @@
+# HTTP 1.1 support
+proxy_http_version 1.1;
+proxy_buffering off;
+proxy_set_header Host $http_host;
+proxy_set_header Upgrade $http_upgrade;
+proxy_set_header Connection $proxy_connection;
+proxy_set_header X-Real-IP $remote_addr;
+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
+proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
+proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
+
+# Mitigate httpoxy attack (see README for details)
+proxy_set_header Proxy "";
+
+# Timeouts for the proxy connection - in sync with the appropriate max_execution time.
+proxy_connect_timeout 3000s;
+proxy_read_timeout 3000s;
+proxy_send_timeout 3000s;

component/web/web.env

@@ -0,0 +1,2 @@
+DEFAULT_HOST=${DEFAULT_HOST}
+HTTPS_METHOD=${HTTPS_METHOD}

component/web/web.go

@@ -0,0 +1,39 @@
+package web
+
+import (
+	"embed"
+
+	"github.com/FAU-CDI/wisski-distillery/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+)
+
+// Web implements the web component
+type Web struct {
+	component.ComponentBase
+}
+
+func (Web) Name() string {
+	return "web"
+}
+
+//go:embed all:stack
+//go:embed web.env
+var resources embed.FS
+
+func (web Web) Stack() stack.Installable {
+	HTTPS_METHOD := "nohttp"
+	if web.Config.HTTPSEnabled() {
+		HTTPS_METHOD = "redirect"
+	}
+
+	return web.MakeStack(stack.Installable{
+		Resources:   resources,
+		ContextPath: "stack",
+		EnvPath:     "web.env",
+
+		EnvContext: map[string]string{
+			"DEFAULT_HOST": web.Config.DefaultDomain,
+			"HTTPS_METHOD": HTTPS_METHOD,
+		},
+	})
+}