Rename packages

internal/wisski/backup.go

@@ -0,0 +1,324 @@
+package wisski
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
+)
+
+// backupDescription is a description for a backup
+type BackupDescription struct {
+	Dest string // destination path
+}
+
+// Snapshot represents the result of generating a snapshot
+type Backup struct {
+	Description BackupDescription
+
+	// Start and End Time of the backup
+	StartTime time.Time
+	EndTime   time.Time
+
+	// various error states, which are ignored when creating the snapshot
+	ErrPanic interface{}
+
+	// SQL and triplestore errors
+	SQLErr error
+	TSErr  error
+
+	// TODO: Make this proper
+	ConfigFileErr       error
+	ConfigFilesManifest map[string]error
+
+	// Snapshots containing instances
+	InstanceListErr   error
+	InstanceSnapshots []Snapshot
+
+	// List of files included
+	Manifest []string
+}
+
+func (backup Backup) String() string {
+	var builder strings.Builder
+	backup.Report(&builder)
+	return builder.String()
+}
+
+// Report writes a report from backup into w
+func (backup Backup) Report(w io.Writer) (int, error) {
+	cw := countwriter.NewCountWriter(w)
+
+	encoder := json.NewEncoder(cw)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(cw, "======= Backup =======\n")
+
+	fmt.Fprintf(cw, "Start: %s\n", backup.StartTime)
+	fmt.Fprintf(cw, "End:   %s\n", backup.EndTime)
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Description =======\n")
+	encoder.Encode(backup.Description)
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Errors =======\n")
+	fmt.Fprintf(cw, "Panic:           %v\n", backup.ErrPanic)
+	fmt.Fprintf(cw, "SQLErr:          %s\n", backup.SQLErr)
+	fmt.Fprintf(cw, "TSErr:           %s\n", backup.TSErr)
+	fmt.Fprintf(cw, "ConfigFileErr:   %s\n", backup.ConfigFileErr)
+	fmt.Fprintf(cw, "InstanceListErr: %s\n", backup.InstanceListErr)
+
+	io.WriteString(cw, "\n")
+
+	io.WriteString(cw, "======= Config Files =======\n")
+	encoder.Encode(backup.ConfigFilesManifest) // TODO: Proper manifest
+
+	io.WriteString(cw, "======= Snapshots =======\n")
+	for _, s := range backup.InstanceSnapshots {
+		io.WriteString(cw, s.String())
+		io.WriteString(cw, "\n")
+	}
+
+	io.WriteString(cw, "======= Manifest =======\n")
+	for _, file := range backup.Manifest {
+		io.WriteString(cw, file+"\n")
+	}
+
+	io.WriteString(cw, "\n")
+
+	return cw.Sum()
+}
+
+func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
+	backup.Description = description
+
+	// catch anything critical that happened during the snapshot
+	defer func() {
+		backup.ErrPanic = recover()
+	}()
+
+	// do the create keeping track of time!
+	logging.LogOperation(func() error {
+		backup.StartTime = time.Now()
+		backup.run(io, dis)
+		backup.EndTime = time.Now()
+
+		return nil
+	}, io, "Writing backup files")
+
+	return
+}
+
+var errBackupSkipFile = errors.New("<file not found>")
+
+func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
+	// create a wait group, and message channel
+	wg := &sync.WaitGroup{}
+	files := make(chan string, 4)
+
+	// backup the sql
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
+		files <- sqlPath
+
+		sql, err := os.Create(sqlPath)
+		if err != nil {
+			backup.SQLErr = err
+			return
+		}
+		defer sql.Close()
+
+		// directly store the result
+		backup.SQLErr = dis.SQL().BackupAll(io, sql)
+	}()
+
+	// backup the triplestore
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		tsPath := filepath.Join(backup.Description.Dest, "triplestore")
+		files <- tsPath
+
+		// directly store the result
+		backup.TSErr = dis.Triplestore().BackupAll(tsPath)
+	}()
+
+	// backup configuration files
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
+		if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
+			backup.ConfigFileErr = err
+			return
+		}
+
+		configs := []string{
+			dis.Config.ConfigPath,
+			filepath.Join(dis.Config.DeployRoot, core.Executable), // TODO: constant the name of the executable
+			dis.Config.SelfOverridesFile,
+			dis.Config.GlobalAuthorizedKeysFile,
+		}
+
+		backup.ConfigFilesManifest = make(map[string]error, len(configs))
+		for _, src := range configs {
+			if !fsx.IsFile(src) {
+				backup.ConfigFilesManifest[src] = errBackupSkipFile
+				continue
+			}
+			dest := filepath.Join(cfgBackupDir, filepath.Base(src))
+
+			// copy the config file and store the error message
+			files <- src
+			backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
+		}
+	}()
+
+	// backup instances
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
+		if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		// list all instances
+		instances, err := dis.AllInstances()
+		if err != nil {
+			backup.InstanceListErr = err
+			return
+		}
+
+		iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
+
+		backup.InstanceSnapshots = make([]Snapshot, len(instances))
+		for i, instance := range instances {
+			backup.InstanceSnapshots[i] = func() Snapshot {
+				dir := filepath.Join(instancesBackupDir, instance.Slug)
+				if err := os.Mkdir(dir, fs.ModeDir); err != nil {
+					return Snapshot{
+						ErrPanic: err,
+					}
+				}
+
+				files <- dir
+				return instance.Snapshot(iochild, SnapshotDescription{
+					Dest: dir,
+				})
+			}()
+		}
+
+	}()
+
+	// wait for the group, then close the message channel.
+	go func() {
+		wg.Wait()
+		close(files)
+	}()
+
+	for file := range files {
+		// get the relative path to the root of the manifest.
+		// nothing *should* go wrong, but in case it does, use the original path.
+		path, err := filepath.Rel(backup.Description.Dest, file)
+		if err != nil {
+			path = file
+		}
+
+		// write it to the command line
+		// and also add it to the manifest
+		io.Printf("\033[2K\r%s", path)
+		backup.Manifest = append(backup.Manifest, path)
+	}
+	slices.Sort(backup.Manifest) // backup the manifest
+	io.Println("")
+
+	// sort the instances manifest
+	slices.SortFunc(backup.InstanceSnapshots, func(a, b Snapshot) bool {
+		return a.Instance.Slug < b.Instance.Slug
+	})
+}
+
+// WriteReport writes out the report belonging to this backup.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (backup Backup) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(backup.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = report.WriteString(backup.String())
+		return err
+	}, io, "Writing backup report")
+}
+
+// ShouldPrune determines if a file with the provided modtime
+func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
+	return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
+}
+
+// PruneBackups prunes all backups older than the maximum backup age
+func (dis *Distillery) PruneBackups(io stream.IOStream) error {
+	sPath := dis.SnapshotsArchivePath()
+
+	// list all the files
+	entries, err := os.ReadDir(sPath)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range entries {
+		// skip directories
+		if entry.IsDir() {
+			continue
+		}
+
+		// grab info about the file
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+
+		// check if it should be pruned!
+		if !dis.ShouldPrune(info.ModTime()) {
+			continue
+		}
+
+		// assemble path, and then remove the file!
+		path := filepath.Join(sPath, entry.Name())
+		io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
+
+		if err := os.Remove(path); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/wisski/component.go

@@ -0,0 +1,124 @@
+package wisski
+
+import (
+	"path/filepath"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/resolver"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/self"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
+	"github.com/FAU-CDI/wisski-distillery/internal/component/web"
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
+)
+
+// components holds the various components of the distillery
+// It is inlined into the [Distillery] struct, and initialized using [makeComponent].
+type components struct {
+	// m protects the fields below
+	m sync.Mutex
+
+	// each component is only initialized once
+	web      *web.Web
+	self     *self.Self
+	resolver *resolver.Resolver
+	dis      *dis.Dis
+	ssh      *ssh.SSH
+	ts       *triplestore.Triplestore
+	sql      *sql.SQL
+}
+
+// makeComponent makes or returns a component inside the [component] struct of the distillery
+//
+// C is the type of component to initialize. It must be backed by a pointer, or makeComponent will panic.
+//
+// dis is the distillery to initialize components for
+// field is a pointer to the appropriate struct field within the distillery components
+// init is called with a new non-nil component to initialize it. It may be nil, to indicate no initialization is required.
+//
+// makeComponent returns the new or existing component instance
+func makeComponent[C component.Component](dis *Distillery, field *C, init func(C)) C {
+	dis.components.m.Lock()
+	defer dis.components.m.Unlock()
+
+	// get the typeof C and make sure that it is a pointer type!
+	typC := reflect.TypeOf((*C)(nil)).Elem()
+	if typC.Kind() != reflect.Pointer {
+		panic("makeComponent: C must be backed by a pointer")
+	}
+
+	// if the component is non-nil, then it has already been initialized
+	if !reflect.ValueOf(*field).IsNil() {
+		return *field
+	}
+
+	// create a new element, and call the initializer (if requested)
+	*field = reflect.New(typC.Elem()).Interface().(C)
+	if init != nil {
+		init(*field)
+	}
+
+	// apply the base configuration
+	base := (*field).Base()
+	base.Config = dis.Config
+	base.Dir = filepath.Join(dis.Config.DeployRoot, "core", (*field).Name())
+
+	// and eventually return it
+	return *field
+}
+
+// Components returns all components of the distillery
+func (dis *Distillery) Components() []component.Component {
+	return []component.Component{
+		dis.Web(),
+		dis.Self(),
+		dis.Resolver(),
+		dis.Dis(),
+		dis.SSH(),
+		dis.Triplestore(),
+		dis.SQL(),
+	}
+}
+
+func (dis *Distillery) Web() *web.Web {
+	return makeComponent(dis, &dis.components.web, nil)
+}
+
+func (dis *Distillery) Self() *self.Self {
+	return makeComponent(dis, &dis.components.self, nil)
+}
+
+func (dis *Distillery) Resolver() *resolver.Resolver {
+	return makeComponent(dis, &dis.components.resolver, func(resolver *resolver.Resolver) {
+		resolver.ConfigName = core.PrefixConfig
+	})
+}
+
+func (d *Distillery) Dis() *dis.Dis {
+	return makeComponent(d, &d.components.dis, nil)
+}
+
+func (dis *Distillery) SSH() *ssh.SSH {
+	return makeComponent(dis, &dis.components.ssh, nil)
+}
+
+func (dis *Distillery) SQL() *sql.SQL {
+	return makeComponent(dis, &dis.components.sql, func(sql *sql.SQL) {
+		sql.ServerURL = dis.Upstream.SQL
+		sql.PollContext = dis.Context()
+		sql.PollInterval = time.Second
+	})
+}
+
+func (dis *Distillery) Triplestore() *triplestore.Triplestore {
+	return makeComponent(dis, &dis.components.ts, func(ts *triplestore.Triplestore) {
+		ts.BaseURL = "http://" + dis.Upstream.Triplestore
+		ts.PollContext = dis.Context()
+		ts.PollInterval = time.Second
+	})
+}

internal/wisski/distillery.go

@@ -0,0 +1,37 @@
+package wisski
+
+import (
+	"context"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+)
+
+// Distillery represents a WissKI Distillery
+//
+// It is the main structure used to interact with different components.
+type Distillery struct {
+	// Config holds the configuration of the distillery.
+	// It is read directly from a configuration file.
+	Config *config.Config
+
+	// Upstream holds information to connect to the various running
+	// distillery components.
+	//
+	// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
+	// But for now this will just hold upstream configuration.
+	Upstream Upstream
+
+	// components hold references to the various components of the distillery.
+	components
+}
+
+// Upstream are the upstream urls connecting to the various external components.
+type Upstream struct {
+	SQL         string
+	Triplestore string
+}
+
+// Context returns a new Context belonging to this distillery
+func (dis *Distillery) Context() context.Context {
+	return context.Background()
+}

internal/wisski/init.go

@@ -0,0 +1,62 @@
+package wisski
+
+import (
+	"os"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+var errNoConfigFile = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Configuration File does not exist",
+}
+
+var errOpenConfig = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "error loading configuration file: %s",
+}
+
+// NewDistillery creates a new distillery from the provided flags
+func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
+	dis = &Distillery{
+		Upstream: Upstream{
+			SQL:         "127.0.0.1:3306",
+			Triplestore: "127.0.0.1:7200",
+		},
+	}
+
+	if flags.InternalInDocker {
+		dis.Upstream.SQL = "sql:3306"
+		dis.Upstream.Triplestore = "triplestore:7200"
+	}
+
+	// if we don't need to load the config, there is nothing to do
+	if !req.NeedsDistillery {
+		return
+	}
+
+	// try to find the configuration file
+	cfg := flags.ConfigPath // command line flags first
+	if cfg == "" {
+		cfg = params.ConfigPath // then globally provided files
+	}
+	if cfg == "" {
+		return nil, errNoConfigFile
+	}
+
+	// open the config file!
+	f, err := os.Open(params.ConfigPath)
+	if err != nil {
+		return nil, errOpenConfig.WithMessageF(err)
+	}
+	defer f.Close()
+
+	// unmarshal the config
+	dis.Config = &config.Config{
+		ConfigPath: cfg,
+	}
+	err = dis.Config.Unmarshal(f)
+	return
+}

internal/wisski/instances.go

@@ -0,0 +1,408 @@
+package wisski
+
+import (
+	"bytes"
+	"embed"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/alessio/shellescape"
+	"github.com/pkg/errors"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+var errNoBookkeeping = exit.Error{
+	Message:  "instance %q does not exist in bookkeeping table",
+	ExitCode: exit.ExitGeneric,
+}
+
+var ErrInstanceNotFound = exit.Error{
+	Message:  "instance not found",
+	ExitCode: exit.ExitGeneric,
+}
+
+var errSQL = exit.Error{
+	Message:  "Unknown SQL Error %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Instance returns the instance of the WissKI Distillery with the provided slug
+func (dis *Distillery) Instance(slug string) (i Instance, err error) {
+	sql := dis.SQL()
+	if err := sql.Wait(); err != nil {
+		return i, err
+	}
+
+	table, err := sql.OpenBookkeeping(false)
+	if err != nil {
+		return i, err
+	}
+
+	// find the instance by slug
+	query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
+	switch {
+	case query.Error != nil:
+		return i, errSQL.WithMessageF(query.Error)
+	case query.RowsAffected == 0:
+		return i, ErrInstanceNotFound
+	default:
+		i.dis = dis
+		return i, nil
+	}
+}
+
+// HasInstance checks if the provided instance exists in the bookeeping table
+func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
+	sql := dis.SQL()
+	if err := sql.Wait(); err != nil {
+		return false, err
+	}
+
+	table, err := sql.OpenBookkeeping(false)
+	if err != nil {
+		return false, err
+	}
+
+	query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok)
+	if query.Error != nil {
+		return false, errSQL.WithMessageF(query.Error)
+	}
+	return
+}
+
+// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances.
+func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) {
+	if len(slugs) == 0 {
+		return dis.AllInstances()
+	}
+	return dis.InstancesWith(slugs...)
+}
+
+// AllInstances returns all instances of the WissKI Distillery in consistent order.
+//
+// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order.
+func (dis *Distillery) AllInstances() ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table
+	})
+}
+
+// InstancesWith returns all instances where the slug is in the provided list of names.
+// The returned instances are reordered in a consistent order.
+func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
+	return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
+		return table.Where("slug IN ?", slugs)
+	})
+}
+
+// findInstances finds instance objects based on a query in the bookkeeping table
+func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
+	sql := dis.SQL()
+	if err := sql.Wait(); err != nil {
+		return nil, err
+	}
+
+	// open the bookkeeping table
+	table, err := sql.OpenBookkeeping(false)
+	if err != nil {
+		return nil, err
+	}
+
+	// prepare a query
+	find := table
+	if order {
+		find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false})
+	}
+	if query != nil {
+		find = query(find)
+	}
+
+	// fetch bookkeeping instances
+	var bks []bookkeeping.Instance
+	find = find.Find(&bks)
+	if find.Error != nil {
+		return nil, errSQL.WithMessageF(find.Error)
+	}
+
+	// make proper instances
+	instances = make([]Instance, len(bks))
+	for i, bk := range bks {
+		instances[i].Instance = bk
+		instances[i].dis = dis
+	}
+
+	return instances, nil
+}
+
+// Instance represents a bookkeeping instance
+type Instance struct {
+	bookkeeping.Instance
+
+	// Credentials for the drupal instance
+	DrupalUsername string
+	DrupalPassword string
+
+	dis *Distillery
+}
+
+// Update updates the bookkeeping table with this instance.
+func (instance *Instance) Update() error {
+	db, err := instance.dis.SQL().OpenBookkeeping(false)
+	if err != nil {
+		return err
+	}
+
+	// it has never been created => we need to create it in the database
+	if instance.Instance.Created.IsZero() {
+		return db.Create(&instance.Instance).Error
+	}
+
+	// Update based on the primary key!
+	return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error
+}
+
+// Delete deletes this instance from the bookkeeping table
+func (instance *Instance) Delete() error {
+	db, err := instance.dis.SQL().OpenBookkeeping(false)
+	if err != nil {
+		return err
+	}
+
+	// doesn't exist => nothing to delete
+	if instance.Instance.Created.IsZero() {
+		return nil
+	}
+
+	// delete it directly
+	return db.Delete(&instance.Instance).Error
+}
+
+// Shell executes a shell command inside the
+func (instance Instance) Shell(io stream.IOStream, argv ...string) (int, error) {
+	return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...)
+}
+
+// Domain returns the full domain name of this instance
+func (instance Instance) Domain() string {
+	return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain)
+}
+
+// URL returns the public URL of this instance
+func (instance Instance) URL() *url.URL {
+	// setup domain and path
+	url := &url.URL{
+		Host: instance.Domain(),
+		Path: "/",
+	}
+
+	// use http or https scheme depending on if the distillery has it enabled
+	if instance.dis.Config.HTTPSEnabled() {
+		url.Scheme = "https"
+	} else {
+		url.Scheme = "http"
+	}
+
+	return url
+}
+
+//go:embed all:instances/barrel instances/barrel.env
+var barrelResources embed.FS
+
+// Stack represents a stack representing this instance
+func (instance Instance) Stack() component.Installable {
+	return component.Installable{
+		Stack: component.Stack{
+			Dir: instance.FilesystemBase,
+		},
+
+		Resources:   barrelResources,
+		ContextPath: filepath.Join("instances", "barrel"),
+		EnvPath:     filepath.Join("instances", "barrel.env"),
+
+		EnvContext: map[string]string{
+			"DATA_PATH": filepath.Join(instance.FilesystemBase, "data"),
+
+			"SLUG":         instance.Slug,
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.Config.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
+
+			"RUNTIME_DIR":                 instance.dis.RuntimeDir(),
+			"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
+		},
+
+		MakeDirsPerm: fs.ModeDir | fs.ModePerm,
+		MakeDirs:     []string{"data", ".composer"},
+
+		TouchFiles: []string{
+			filepath.Join("data", "authorized_keys"),
+		},
+	}
+}
+
+//go:embed all:instances/reserve instances/reserve.env
+var reserveResources embed.FS
+
+func (instance Instance) ReserveStack() component.Installable {
+	return component.Installable{
+		Stack: component.Stack{
+			Dir: instance.FilesystemBase,
+		},
+
+		Resources:   reserveResources,
+		ContextPath: filepath.Join("instances", "reserve"),
+		EnvPath:     filepath.Join("instances", "reserve.env"),
+
+		EnvContext: map[string]string{
+			"VIRTUAL_HOST": instance.Domain(),
+
+			"LETSENCRYPT_HOST":  instance.dis.Config.IfHttps(instance.Domain()),
+			"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
+		},
+	}
+}
+
+// Provision provisions an instance, assuming that the required databases already exist.
+func (instance Instance) Provision(io stream.IOStream) error {
+
+	// create the basic st!
+	st := instance.Stack()
+	if err := st.Install(io, component.InstallationContext{}); err != nil {
+		return err
+	}
+
+	// Pull and build the stack!
+	if err := st.Update(io, false); err != nil {
+		return err
+	}
+
+	provisionParams := []string{
+		instance.Domain(),
+
+		instance.SqlDatabase,
+		instance.SqlUser,
+		instance.SqlPassword,
+
+		instance.GraphDBRepository,
+		instance.GraphDBUser,
+		instance.GraphDBPassword,
+
+		instance.DrupalUsername,
+		instance.DrupalPassword,
+
+		"", // TODO: DrupalVersion
+		"", // TODO: WissKIVersion
+	}
+
+	// escape the parameter
+	for i, param := range provisionParams {
+		provisionParams[i] = shellescape.Quote(param)
+	}
+
+	// figure out the provision script
+	// TODO: Move the provision script into the control plane!
+	provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
+
+	code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errors.New("Unable to run provision script")
+	}
+
+	return nil
+}
+
+func (instance *Instance) NoPrefix() bool {
+	return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip"))
+}
+
+var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
+
+// PrefixConfig returns the prefix config belonging to this instance.
+func (instance *Instance) PrefixConfig() (config string, err error) {
+	// if the user requested to skip the prefix, then don't do anything with it!
+	if instance.NoPrefix() {
+		return "", nil
+	}
+
+	var builder strings.Builder
+
+	// domain
+	builder.WriteString(instance.URL().String() + ":")
+	builder.WriteString("\n")
+
+	// default prefixes
+	wu := stream.NewIOStream(&builder, nil, nil, 0)
+	code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
+	if err != nil || code != 0 {
+		return "", errPrefixExecFailed
+	}
+
+	// custom prefixes
+	prefixPath := filepath.Join(instance.FilesystemBase, "prefixes")
+	if fsx.IsFile(prefixPath) {
+		prefix, err := os.Open(prefixPath)
+		if err != nil {
+			return "", err
+		}
+		defer prefix.Close()
+		if _, err := io.Copy(&builder, prefix); err != nil {
+			return "", err
+		}
+		builder.WriteString("\n")
+	}
+
+	// and done!
+	return builder.String(), nil
+}
+
+var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder")
+
+// ExportPathbuilders writes pathbuilders into the directory dest
+func (instance *Instance) ExportPathbuilders(dest string) error {
+	// export all the pathbuilders into the buffer
+	var buffer bytes.Buffer
+	wu := stream.NewIOStream(&buffer, nil, nil, 0)
+	code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php")
+	if err != nil || code != 0 {
+		return errPathbuildersExecFailed
+	}
+
+	// decode them as a json array
+	var pathbuilders map[string]string
+	if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil {
+		return err
+	}
+
+	// sort the names of the pathbuilders
+	names := maps.Keys(pathbuilders)
+	slices.Sort(names)
+
+	// write each into a file!
+	for _, name := range names {
+		pbxml := []byte(pathbuilders[name])
+		name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
+		if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/wisski/instances/barrel.env

@@ -0,0 +1,10 @@
+DATA_PATH=${DATA_PATH}
+RUNTIME_DIR=${RUNTIME_DIR}
+
+SLUG=${SLUG}
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
+
+GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}

internal/wisski/instances/barrel/.dockerignore

@@ -0,0 +1,8 @@
+# Ignore everything
+*
+
+# allow the following files:
+!conf/*
+!scripts/*
+!patch/*
+!wisskiutils/*

internal/wisski/instances/barrel/.env.sample

@@ -0,0 +1,28 @@
+#######################
+# Meta Settings
+#######################
+
+# Real path for volumes to be stored
+DATA_PATH=/var/www/deploy/instances/example.slug/data
+UTILS_DIR=/var/www/deploy/runtime/utils/
+
+#######################
+### Web Server settings
+#######################
+# the hostname for the website
+VIRTUAL_HOST=example.com
+
+# optional letsencrypt support
+# when blank, ignore
+LETSENCRYPT_HOST=
+LETSENCRYPT_EMAIL=
+
+### SQL settings
+MYSQL_HOST=mysql
+MYSQL_USER=user
+MYSQL_PASS=pass
+
+### GraphDB settings
+GRAPHDB_HOST=graphdb
+GRAPHDB_USER=user
+GRAPHDB_PASS=pass

internal/wisski/instances/barrel/Dockerfile

@@ -0,0 +1,106 @@
+FROM docker.io/library/php:8.0-apache-bullseye
+ARG COMPOSER_VERSION=2.3.8
+WORKDIR /var/www
+
+# install and enable the various required php extension
+RUN apt-get update && apt-get install -y \
+        curl \
+        default-mysql-client \
+        git \
+        imagemagick \
+        libcurl4-openssl-dev \
+        libfreetype6-dev \
+        libicu-dev \
+        libjpeg62-turbo-dev \
+        libpng-dev \
+        libssh2-1-dev \
+        libwebp-dev \
+        libxml2-dev \
+        libxpm-dev \
+        sudo \
+        unzip \
+        vim \
+        zip \
+    && \
+    docker-php-source extract && \
+    mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" && \
+    pear config-set php_ini "$PHP_INI_DIR/php.ini" && \
+    docker-php-ext-configure gd \
+        --enable-gd \
+        --with-webp \
+        --with-jpeg \
+        --with-xpm \
+        --with-freetype \
+        --enable-gd-jis-conv \
+    && \
+    docker-php-ext-install \
+        curl \
+        gd \
+        intl \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        soap \
+        xml \
+    && \
+    pecl install xmlrpc-1.0.0RC3 && \
+    pecl install ssh2-1.3.1 && \
+    pecl install apcu-5.1.21 && \
+    pecl install uploadprogress-2.0.2 && \
+    docker-php-ext-enable \
+        apcu \
+        curl \
+        gd \
+        intl \
+        mysqli \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        soap \
+        ssh2 \
+        uploadprogress \
+        xml \
+        xmlrpc \
+    && \
+    docker-php-source delete
+
+# enable the apache rewrite mod
+RUN a2enmod rewrite
+
+# install composer and add it to path
+RUN curl -sS https://getcomposer.org/installer | php -- --version=$COMPOSER_VERSION && \
+    mv composer.phar /usr/local/bin/composer
+ENV PATH "/usr/local/bin:/var/www/data/project/vendor/bin:$PATH"
+
+# remove default configuration
+RUN rm /etc/apache2/sites-available/*.conf && \
+    rm /etc/apache2/sites-enabled/*.conf
+
+ADD patch/easyrdf.patch /patch/easyrdf.patch
+ADD patch/triples.patch /patch/triples.patch
+
+# Add wisski configuration
+ADD conf/ports.conf /etc/apache2/ports.conf
+ADD conf/wisski.conf /etc/apache2/sites-available/wisski.conf
+ADD conf/wisski.ini /usr/local/etc/php/conf.d/wisski.ini
+RUN a2ensite wisski
+
+# volumes for composer
+VOLUME /var/www/.composer
+VOLUME /var/www/data
+
+# Add and configure the entrypoint
+ADD scripts/entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
+CMD ["apache2-foreground"]
+
+# Add the provision script and WissKI utils
+ADD scripts/provision_container.sh /provision_container.sh
+ADD wisskiutils/ /wisskiutils
+
+# Add the user_shell.sh
+ADD scripts/user_shell.sh /user_shell.sh
+
+# expose port 8080
+EXPOSE 8080

internal/wisski/instances/barrel/conf/ports.conf

@@ -0,0 +1,4 @@
+# This file configures where apache should listen. 
+# Because we are running as a limited user, we want to listen on a high port. 
+# For this we use port 8080
+Listen 8080

internal/wisski/instances/barrel/conf/wisski.conf

@@ -0,0 +1,24 @@
+<VirtualHost *:8080>
+    # the document root -- /var/www/data/project/web
+    DocumentRoot /var/www/data/project/web
+
+    <Directory /var/www/data/project/web>
+        # add types for .owl and .rdf
+        AddType application/rdf+xml .owl
+        AddType application/rdf+xml .rdf
+
+        # Rewrite the 'ontology' directory
+        RewriteEngine On
+        RewriteOptions InheritDownBefore
+        ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,END]
+        ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [END]
+
+        # Allow overrides of symlinks
+        Options Indexes FollowSymLinks
+        AllowOverride All
+        Require all granted
+    </Directory>
+
+    ErrorLog /dev/stderr
+    CustomLog /dev/stdout combined
+</VirtualHost>

internal/wisski/instances/barrel/conf/wisski.ini

@@ -0,0 +1,14 @@
+; File Uploads up to 1GB
+file_uploads = On
+upload_max_filesize = 1000M
+post_max_size = 1000M
+
+; Composer uses an absurd amount of memory
+; 4GB ought to be enough
+memory_limit = 4G
+
+; Increase various limits for some long running WissKI operations
+max_execution_time = 3000
+max_input_time = 600
+max_input_nesting_level = 640
+max_input_vars = 10000

internal/wisski/instances/barrel/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.7"
+
+services:
+  barrel:
+    build: .
+    restart: always
+    hostname: ${VIRTUAL_HOST}.wisski
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8080
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+    
+    # label it with the current slug
+    labels:
+      eu.wiss-ki.barrel.slug: ${SLUG}
+      eu.wiss-ki.barrel.authfile: /var/www/.ssh/authorized_keys,/var/www/.ssh/global_authorized_keys
+    
+    # volumes that are mounted
+    volumes:
+      - ${GLOBAL_AUTHORIZED_KEYS_FILE}:/var/www/.ssh/global_authorized_keys:ro
+      - ${DATA_PATH}/.composer:/var/www/.composer
+      - ${DATA_PATH}/data:/var/www/data
+      - ${DATA_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
+      - ${RUNTIME_DIR}:/runtime:ro
+
+networks:
+  default:
+    name: distillery
+    external: true

internal/wisski/instances/barrel/patch/easyrdf.patch

@@ -0,0 +1,4 @@
+281c281
+<         if (preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]+)|", $status, $m)) {
+---
+>         if(preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]*)|", $status, $m)) {

internal/wisski/instances/barrel/patch/triples.patch

@@ -0,0 +1,8 @@
+100c100
+<             if($result->o instanceof \EasyRdf_Resource) {
+---
+> 	    if($result->o instanceof \EasyRdf\Resource) {
+118c118
+<               $object_text = $result->o->getValue();
+---
+>               $object_text = $result->o->dumpValue('string');

internal/wisski/instances/barrel/scripts/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script contains 
+
+# chown the volumes to make sure they can be read and written by the limited user
+chown www-data:www-data /var/www
+chown www-data:www-data /var/www/.composer
+chown www-data:www-data /var/www/data/
+
+# run the original entrypoint
+docker-php-entrypoint "$@"

internal/wisski/instances/barrel/scripts/provision_container.sh

@@ -0,0 +1,174 @@
+#!/bin/bash
+set -e
+
+function log_info() {
+   echo -e "\033[1m$1\033[0m"
+}
+
+function log_ok() {
+   echo -e "\033[0;32m$1\033[0m"
+}
+
+log_info " => Reading configuration variables"
+
+INSTANCE_DOMAIN="$1"
+echo "INSTANCE_DOMAIN=$INSTANCE_DOMAIN"
+shift 1
+
+MYSQL_DATABASE="$1"
+echo "MYSQL_DATABASE=$MYSQL_DATABASE"
+MYSQL_USER="$2"
+echo "MYSQL_USER=$MYSQL_USER"
+MYSQL_PASSWORD="$3"
+echo "MYSQL_PASSWORD=$MYSQL_PASSWORD"
+
+shift 3
+
+GRAPHDB_REPO="$1"
+echo "GRAPHDB_REPO=$GRAPHDB_REPO"
+GRAPHDB_USER="$2"
+echo "GRAPHDB_USER=$GRAPHDB_USER"
+GRAPHDB_PASSWORD="$3"
+echo "GRAPHDB_PASSWORD=$GRAPHDB_PASSWORD"
+shift 3
+
+GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
+
+DRUPAL_USER="$1"
+echo "DRUPAL_USER=$DRUPAL_USER"
+DRUPAL_PASS="$2"
+echo "DRUPAL_PASS=$DRUPAL_PASS"
+shift 2
+
+DRUPAL_VERSION="$1"
+echo "DRUPAL_VERSION=$DRUPAL_VERSION"
+shift 1
+
+WISSKI_VERSION="$1"
+echo "WISSKI_VERSION=$WISSKI_VERSION"
+shift 1
+
+log_info " => Preparing installation environment"
+BASE_DIR="/var/www/data"
+COMPOSER_DIR="$BASE_DIR/project"
+WEB_DIR="$COMPOSER_DIR/web"
+ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
+
+log_info " => Creating '$COMPOSER_DIR'"
+mkdir -p "$COMPOSER_DIR"
+cd "$COMPOSER_DIR"
+
+# workaround for making the drupal sites directory writable
+function drupal_sites_permission_workaround() {
+    chmod -R u+w "$WEB_DIR/sites/" || true
+}
+
+# install a module with composer and enable it with drush
+# Example:
+#
+# composer_install_and_enable << EOF
+# drupal/some_module:1.23 some_module
+# drupal/other_module:2.34
+# EOF
+# 
+# Will install both modules, but only enable the first one.
+function composer_install_and_enable() {
+    while IFS= read -r line; do
+        echo "$line" | (
+            read composer drush;
+            drupal_sites_permission_workaround
+            composer require "$composer"
+            if [ -n "$drush" ]; then
+                drush pm-enable --yes "$drush"
+            fi
+        )
+    done
+}
+
+
+# Create a new composer project. 
+log_info " => Creating composer project"
+if [ -z "${DRUPAL_VERSION}" ]; then
+    composer --no-interaction create-project 'drupal/recommended-project:^9.0.0' .
+else
+    composer --no-interaction create-project "drupal/recommended-project:$DRUPAL_VERSION" .
+fi
+
+# needed for composer > 2.2
+composer --no-interaction config allow-plugins true
+
+# Install drush so that we can automate a lot of things
+log_info " => Installing 'drush'"
+composer require drush/drush
+
+# Use 'drush' to run the site-installation. 
+# Here we need to use the username, password and database creds we made above. 
+log_info " => Running drupal installation scripts"
+drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
+    --account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
+    --db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@sql/${MYSQL_DATABASE}
+drupal_sites_permission_workaround
+
+# create a directory for ontologies. 
+log_info " => Creating '$ONTOLOGY_DIR'"
+mkdir -p "$ONTOLOGY_DIR"
+
+# Install the Wisski packages. 
+log_info " => Installing Wisski packages"
+cd "$COMPOSER_DIR"
+
+# install the development version when requested
+if [ -z "${WISSKI_VERSION}" ]; then
+    composer require 'drupal/wisski'
+else
+    composer require "drupal/wisski:$WISSKI_VERSION"
+fi
+
+# Install dependencies of WissKI
+log_info " => Installing and patching Wisski dependencies"
+pushd "$WEB_DIR/modules/contrib/wisski"
+composer install
+
+# Patch EasyRDF (for now)
+EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
+if [ -f "$EASYRDF_RESPONSE" ]; then
+    patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"
+fi
+popd
+
+log_info " => Installing and enabling additional modules"
+composer_install_and_enable << EOF
+drupal/inline_entity_form:^1.0@RC
+drupal/imagemagick
+drupal/image_effects
+drupal/colorbox
+drupal/devel:^4.1 devel
+drupal/geofield:^1.40 geofield
+drupal/geofield_map:^2.85 geofield_map
+drupal/imce:^2.4 imce
+EOF
+
+log_info " => Enable Wisski modules"
+drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
+drupal_sites_permission_workaround
+
+log_info " => Setting up WissKI Salz Adapter"
+drush php:script /wisskiutils/create_adapter.php "$INSTANCE_DOMAIN" "$GRAPHDB_REPO" "$GRAPHDB_HEADER"
+
+log_info " => Updating TRUSTED_HOST_PATTERNS in settings.php"
+
+/bin/bash /wisskiutils/set_trusted_host.sh
+
+log_info " => Running initial cron"
+drush core-cron
+
+log_info " => Provisioning is now complete. "
+log_ok "Your installation details are as follows:"
+function printdetails() {
+    echo "URL:                  http://$INSTANCE_DOMAIN"
+    echo "Username:             $DRUPAL_USER"
+    echo "Password:             $DRUPAL_PASS"
+}
+printdetails
+
+exit 0

internal/wisski/instances/barrel/scripts/user_shell.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# This script is used to start a user shell inside the docker container. 
+cd "/var/www/data/project"
+sudo -u www-data "PATH=/var/www/data/project/vendor/bin:$PATH" /bin/bash "$@"

internal/wisski/instances/barrel/wisskiutils/create_adapter.php

@@ -0,0 +1,61 @@
+<?php
+
+/**
+ * This script will automatically create a WissKI Salz Adapter for use within the distillery.
+ * It will not update any existing adapter and is rather primitive.
+ */
+
+$argc = $_SERVER['argc']-3;
+$argv = array_slice($_SERVER['argv'], 3);
+
+// read parameters from the command line
+if ($argc != 3) {
+    die("Usage: drush php:script create_adapter.php INSTANCE_DOMAIN GRAPHDB_REPO HEADER");
+}
+$INSTANCE_DOMAIN = $argv[0];
+$GRAPHDB_REPO = $argv[1];
+$HEADER = $argv[2];
+
+//
+// PROPERTIES FOR THE ADAPTER
+//
+
+$id = 'default'; // id
+$type = 'sparql11_with_pb'; // plugin
+$machine_name = 'default'; // machine-name
+$label = 'Default WissKI Distillery Adapter';
+$description = 'Adapter for ' . $INSTANCE_DOMAIN; // description
+$writable = TRUE; // writable
+$is_preferred_local_store = TRUE; // is_preferred_local_store
+$header = $HEADER; // header
+$read_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO; // read_url
+$write_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO . '/statements'; // write_url
+$is_federatable = TRUE; // is_federatable
+$default_graph_uri = 'https://' . $INSTANCE_DOMAIN . '/';
+$same_as_properties = ['http://www.w3.org/2002/07/owl#sameAs']; // same_as_properties
+$ontology_graphs = []; // ontology_graphs
+
+//
+// Do the creation!
+//
+
+$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+$adapter = $storage->create([
+    "id" => $id,
+    "label" => $label,
+    "description" => $description,
+]);
+$adapter->setEngineConfig([
+    "id" => $type,
+    "machine-name" => $machine_name,
+    "header" => $header,
+    "writeable" => $writable,
+    "is_preferred_local_store" => $is_preferred_local_store,
+    "read_url" => $read_url,
+    "write_url" => $write_url,
+    "is_federatable" => $is_federatable,
+    "default_graph" => $default_graph_uri,
+    "same_as_properties" => $same_as_properties,
+    "ontology_graphs" => $ontology_graphs,
+]);
+$adapter->save();

internal/wisski/instances/barrel/wisskiutils/export_pathbuilder.php

@@ -0,0 +1,63 @@
+<?php
+
+/**
+ * This script will list all the URIs that this system is aware of.
+ * This works by listing all the default graph uris of all the adapters.
+ */
+
+use Drupal\wisski_pathbuilder\Entity\WisskiPathEntity;
+
+// load all the pathbuilders
+$pbs = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->loadMultiple();
+
+// map over the pathbuilders
+$xmls = array_map(function($pb) {
+    $xml = new \SimpleXMLElement("<pathbuilderinterface></pathbuilderinterface>");
+
+    $paths = $pb->getAllPaths();
+    foreach ($paths as $key => $path) {
+        $id = $path->getID();
+
+        $path = $pb->getPbPath($id);
+
+        $pathChild = $xml->addChild("path");
+        $pathObject = WisskiPathEntity::load($id);
+
+        foreach ($path as $subkey => $value) {
+
+            if (in_array($subkey, ['relativepath'])) {
+                continue;
+            }
+
+            if ($subkey == "parent") {
+                $subkey = "group_id";
+            }
+
+            $pathChild->addChild($subkey, htmlspecialchars($value));
+        }
+
+        $pathArray = $pathChild->addChild('path_array');
+        foreach ($pathObject->getPathArray() as $subkey => $value) {
+            $pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value);
+        }
+
+        $pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty()));
+        $pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName()));
+        $pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb()));
+        $pathChild->addChild('description', htmlspecialchars($pathObject->getDescription()));
+        $pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid()));
+        if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") {
+            $pathChild->addChild('is_group', "1");
+        } else {
+            $pathChild->addChild('is_group', "0");
+        }
+        $pathChild->addChild('name', htmlspecialchars($pathObject->getName()));
+    }
+
+    // turn it into XML
+    $dom = dom_import_simplexml($xml)->ownerDocument;
+    $dom->formatOutput = TRUE;
+    return $dom->saveXML();
+}, $pbs);
+
+echo json_encode($xmls);

internal/wisski/instances/barrel/wisskiutils/list_uri_prefixes.php

@@ -0,0 +1,19 @@
+<?php
+
+/**
+ * This script will list all the URIs that this system is aware of.
+ * This works by listing all the default graph uris of all the adapters.
+ */
+
+// iterate over all adapters
+$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+foreach ($storage->loadMultiple() as $adapter) {
+    // read the configuration, and check if we have a default graph
+    $conf = $adapter->getEngine()->getConfiguration();
+    if(!array_key_exists('default_graph', $conf)) {
+        continue;
+    }
+
+    // and echo it out
+    echo $conf['default_graph'] . "\n";
+}

internal/wisski/instances/barrel/wisskiutils/set_trusted_host.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This utility script can be used to configure the trusted host settings inside of settings.php. 
+# It doesn't take care of corner cases and should only be used when needed. 
+
+INSTANCE_DOMAIN="$(hostname -f)"
+INSTANCE_DOMAIN="${INSTANCE_DOMAIN%.wisski}"
+
+TRUSTED_HOST_PATTERN="${INSTANCE_DOMAIN//\./\\\\.}"
+TRUSTED_HOST_PATTERNS='["'$TRUSTED_HOST_PATTERN'"]'
+
+echo "Setting 'trusted_host_patterns' to $TRUSTED_HOST_PATTERNS"
+bash /wisskiutils/settings_php_set.sh 'trusted_host_patterns' "$TRUSTED_HOST_PATTERNS"

internal/wisski/instances/barrel/wisskiutils/settings_php_get.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# settings_php_get.sh name
+# Gets the 'settings_php_get.php' setting 'name' as json-encoded value, or null when it does not exist.
+
+NAME=$1
+
+if [ -z "$NAME" ]; then
+    echo "Usage: get_settings_setting.sh NAME"
+    exit 1
+fi;
+
+echo "$NAME" | drush php:eval '
+  use \Drupal\Core\Site\Settings;
+  $name=trim(file_get_contents("php://stdin"));
+  echo json_encode(Settings::get($name));
+';

internal/wisski/instances/barrel/wisskiutils/settings_php_set.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# settings_php_set.sh name value
+# Sets the 'settings.php' setting 'name' to 'value'.
+# Value must be json-encoded. 
+
+NAME=$1
+VALUE=$2
+
+if [ -z "$NAME" ]; then
+    echo "Usage: settings_php_set.sh NAME VALUE"
+    exit 1
+fi;
+
+if [ -z "$VALUE" ]; then
+    echo "Usage: settings_php_set.sh NAME VALUE"
+    exit 1
+fi;
+
+cd /var/www/data/project
+chmod u+w web/sites/default/settings.php
+
+(echo "$NAME"; echo "$VALUE" ) | drush php:eval '
+    include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
+
+    // read NAME and VALUE from STDIN
+    $content=file_get_contents("php://stdin");  
+    $newline=strpos($content, "\n");
+    $name=trim(substr($content, 0, $newline));
+    $jvalue=trim(substr($content, $newline + 1));
+
+    // decode json values
+    $value = @json_decode($jvalue);
+    if ($value === null && json_last_error() !== JSON_ERROR_NONE) {
+        echo "Invalid JSON, cannot update settings.php. \n";
+        return 1;
+    }
+
+    // make parameters to drush_rewrite_settings
+    $settings["settings"][$name] = (object)[
+        "value" => $value,
+        "required" => TRUE,
+    ];
+
+    // find the actual settings.php file to rewrite
+    $filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
+    drupal_rewrite_settings($settings, $filename);
+
+    echo "Wrote " . $filename . "\n";
+    return 0;   
+';
+EXIT=$?
+
+chmod u-w web/sites/default/settings.php
+
+exit $?

internal/wisski/instances/reserve.env

@@ -0,0 +1,4 @@
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}

internal/wisski/instances/reserve/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.7"
+
+services:
+  static:
+    image: tkw01536/gostatic
+    restart: always
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8043
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+    
+    ports:
+      - 8043
+    
+    # volumes that are mounted
+    volumes:
+      - ./index.html:/srv/http/index.html:ro
+
+networks:
+  default:
+    name: distillery
+    external: true

internal/wisski/instances/reserve/index.html

@@ -0,0 +1,4 @@
+<!DOCTYPE html>
+
+This domain name is reserved. 
+Content is a work in progress. 

internal/wisski/instances_provision.go

@@ -0,0 +1,90 @@
+package wisski
+
+import (
+	"path/filepath"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
+	"github.com/pkg/errors"
+)
+
+func (dis *Distillery) InstancesDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "instances")
+}
+
+func (dis *Distillery) InstanceDir(slug string) string {
+	return filepath.Join(dis.InstancesDir(), slug)
+}
+
+func (dis *Distillery) InstanceSQL(slug string) (database, user string) {
+	database = dis.Config.MysqlDatabasePrefix + slug
+	user = dis.Config.MysqlUserPrefix + slug
+	return
+}
+
+func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) {
+	repo = dis.Config.GraphDBRepoPrefix + slug
+	user = dis.Config.GraphDBUserPrefix + slug
+	return
+}
+
+var errInvalidSlug = errors.New("Not a valid slug")
+
+// NewInstance fills the struct for a new distillery instance.
+// It validates that slug is a valid name for an instance.
+//
+// It does not perform any checks if the instance already exists, or does the creation in the database.
+func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
+
+	// make sure that the slug is valid!
+	if _, err := stringparser.ParseSlug(slug); err != nil {
+		return i, errInvalidSlug
+	}
+
+	// generate sql data
+	sqlPassword, err := dis.Config.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	sqlDB, sqlUser := dis.InstanceSQL(slug)
+
+	// generate ts data
+	tsPassword, err := dis.Config.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	tsRepo, tsUser := dis.InstanceGraphDB(slug)
+
+	// generate drupal data
+	drPassword, err := dis.Config.NewPassword()
+	if err != nil {
+		return i, err
+	}
+	drUser := "admin"
+
+	// make the instance object!
+	instance := bookkeeping.Instance{
+		Slug: slug,
+
+		OwnerEmail:             "",
+		AutoBlindUpdateEnabled: true,
+
+		FilesystemBase: dis.InstanceDir(slug),
+
+		SqlDatabase: sqlDB,
+		SqlUser:     sqlUser,
+		SqlPassword: sqlPassword,
+
+		GraphDBRepository: tsRepo,
+		GraphDBUser:       tsUser,
+		GraphDBPassword:   tsPassword,
+	}
+
+	i.DrupalUsername = drUser
+	i.DrupalPassword = drPassword
+
+	// store the instance in the object and return it!
+	i.Instance = instance
+	i.dis = dis
+	return i, nil
+}

internal/wisski/runtime.go

@@ -0,0 +1,8 @@
+package wisski
+
+import "path/filepath"
+
+// RuntimeDir returns the path to the runtime directory
+func (dis *Distillery) RuntimeDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "runtime")
+}

internal/wisski/server.go

@@ -0,0 +1,33 @@
+package wisski
+
+import (
+	"io"
+	"net/http"
+)
+
+// TODO: Move this into dis!
+
+// Server represents a server for this distillery
+type Server struct {
+	dis *Distillery
+}
+
+func (dis *Distillery) Server() *Server {
+	return &Server{
+		dis: dis,
+	}
+}
+
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	instances, err := s.dis.AllInstances()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, "Something went wrong")
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	for _, instance := range instances {
+		io.WriteString(w, instance.Slug+"\n")
+	}
+}

internal/wisski/snapshot.go

@@ -0,0 +1,330 @@
+package wisski
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
+	"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/FAU-CDI/wisski-distillery/pkg/opgroup"
+	"github.com/FAU-CDI/wisski-distillery/pkg/password"
+	"github.com/tkw1536/goprogram/stream"
+	"golang.org/x/exp/slices"
+)
+
+// SnapshotsDir returns the path that contains all snapshot related data.
+func (dis *Distillery) SnapshotsDir() string {
+	return filepath.Join(dis.Config.DeployRoot, "snapshots")
+}
+
+// SnapshotsStagingPath returns the path to the directory containing a temporary staging area for snapshots.
+// Use NewSnapshotStagingDir to generate a new staging area.
+func (dis *Distillery) SnapshotsStagingPath() string {
+	return filepath.Join(dis.SnapshotsDir(), "staging")
+}
+
+// SnapshotsArchivePath returns the path to the directory containing all exported archives.
+// Use NewSnapshotArchivePath to generate a path to a new archive in this directory.
+func (dis *Distillery) SnapshotsArchivePath() string {
+	return filepath.Join(dis.SnapshotsDir(), "archives")
+}
+
+// NewSnapshotArchivePath returns the path to a new archive with the provided prefix.
+// The path is guaranteed to not exist.
+func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
+	// TODO: Consider moving these into a subdirectory with the provided prefix.
+	for path == "" || fsx.Exists(path) {
+		name := dis.newSnapshotName(prefix) + ".tar.gz"
+		path = filepath.Join(dis.SnapshotsArchivePath(), name)
+	}
+	return
+}
+
+// newSnapshot name returns a new basename for a snapshot with the provided prefix.
+// The name is guaranteed to be unique within this process.
+func (*Distillery) newSnapshotName(prefix string) string {
+	suffix, _ := password.Password(64) // silently ignore any errors!
+	if prefix == "" {
+		prefix = "backup"
+	} else {
+		prefix = "snapshot-" + prefix
+	}
+	return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
+}
+
+// NewSnapshotStagingDir returns the path to a new snapshot directory.
+// The directory is guaranteed to have been freshly created.
+func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
+	for path == "" || os.IsExist(err) {
+		path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
+		err = os.Mkdir(path, os.ModeDir)
+	}
+	if err != nil {
+		path = ""
+	}
+	return
+}
+
+// SnapshotDescription is a description for a snapshot
+type SnapshotDescription struct {
+	Dest      string // destination path
+	Keepalive bool   // should we keep the instance alive while making the snapshot?
+}
+
+// Snapshot represents the result of generating a snapshot
+type Snapshot struct {
+	Description SnapshotDescription
+	Instance    bookkeeping.Instance
+
+	// Start and End Time of the snapshot
+	StartTime time.Time
+	EndTime   time.Time
+
+	// Generic Panic that may have occured
+	ErrPanic interface{}
+
+	// Errors during starting and stopping the system
+	ErrStart error
+	ErrStop  error
+
+	// List of files included
+	Manifest []string
+
+	// Errors during other parts
+	ErrBookkeep    error
+	ErrPathbuilder error
+	ErrFilesystem  error
+	ErrTriplestore error
+	ErrSQL         error
+}
+
+func (snapshot Snapshot) String() string {
+	var builder strings.Builder
+	snapshot.Report(&builder)
+	return builder.String()
+}
+
+// Report writes a report from snapshot into w
+func (snapshot Snapshot) Report(w io.Writer) (int, error) {
+	ww := countwriter.NewCountWriter(w)
+
+	// TODO: Errors of the writer!
+	encoder := json.NewEncoder(ww)
+	encoder.SetIndent("", "  ")
+
+	io.WriteString(ww, "======= Begin Snapshot Report "+snapshot.Instance.Slug+" =======\n")
+
+	fmt.Fprintf(ww, "Slug:  %s\n", snapshot.Instance.Slug)
+	fmt.Fprintf(ww, "Dest:  %s\n", snapshot.Description.Dest)
+
+	fmt.Fprintf(ww, "Start: %s\n", snapshot.StartTime)
+	fmt.Fprintf(ww, "End:   %s\n", snapshot.EndTime)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Description =======\n")
+	encoder.Encode(snapshot.Description)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Instance =======\n")
+	encoder.Encode(snapshot.Instance)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Errors =======\n")
+	fmt.Fprintf(ww, "Panic:       %v\n", snapshot.ErrPanic)
+	fmt.Fprintf(ww, "Start:       %s\n", snapshot.ErrStart)
+	fmt.Fprintf(ww, "Stop:        %s\n", snapshot.ErrStop)
+	fmt.Fprintf(ww, "Bookkeep:    %s\n", snapshot.ErrBookkeep)
+	fmt.Fprintf(ww, "Pathbuilder: %s\n", snapshot.ErrPathbuilder)
+	fmt.Fprintf(ww, "Filesystem:  %s\n", snapshot.ErrFilesystem)
+	fmt.Fprintf(ww, "Triplestore: %s\n", snapshot.ErrTriplestore)
+	fmt.Fprintf(ww, "SQL:         %s\n", snapshot.ErrSQL)
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= Manifest =======\n")
+	for _, file := range snapshot.Manifest {
+		io.WriteString(ww, file+"\n")
+	}
+
+	io.WriteString(ww, "\n")
+
+	io.WriteString(ww, "======= End Snapshot Report "+snapshot.Instance.Slug+"=======\n")
+
+	return ww.Sum()
+}
+
+// Snapshot creates a new snapshot of this instance into dest
+func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
+	// setup the snapshot
+	snapshot.Description = desc
+	snapshot.Instance = instance.Instance
+
+	// capture anything critical, and write the end time
+	defer func() {
+		snapshot.ErrPanic = recover()
+	}()
+
+	// do the create keeping track of time!
+	logging.LogOperation(func() error {
+		snapshot.StartTime = time.Now()
+
+		snapshot.makeBlackbox(io, instance)
+		snapshot.makeWhitebox(io, instance)
+
+		snapshot.EndTime = time.Now()
+		return nil
+	}, io, "Writing snapshot files")
+
+	slices.Sort(snapshot.Manifest)
+	return
+}
+
+// makeBlackbox runs the blackbox backup of the system.
+// It pauses the Instance, if a consistent state is required.
+func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, instance Instance) {
+	stack := instance.Stack()
+
+	og := opgroup.NewOpGroup[string](4)
+
+	// stop the instance (unless it was explicitly asked to not do so!)
+	if !snapshot.Description.Keepalive {
+		logging.LogMessage(io, "Stopping instance")
+		snapshot.ErrStop = stack.Down(io)
+
+		defer func() {
+			logging.LogMessage(io, "Starting instance")
+			snapshot.ErrStart = stack.Up(io)
+		}()
+	}
+
+	// write bookkeeping information
+	og.GoErr(func(files chan<- string) error {
+		bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
+		files <- bkPath
+
+		info, err := os.Create(bkPath)
+		if err != nil {
+			return err
+		}
+		defer info.Close()
+
+		// print whatever is in the database
+		// TODO: This should be sql code, maybe gorm can do that?
+		_, err = fmt.Fprintf(info, "%#v\n", instance.Instance)
+		return err
+	}, &snapshot.ErrBookkeep)
+
+	// backup the filesystem
+	og.GoErr(func(files chan<- string) error {
+		fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
+
+		// copy over whatever is in the base directory
+		return fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
+			files <- dst
+		})
+	}, &snapshot.ErrFilesystem)
+
+	// backup the graph db repository
+	og.GoErr(func(files chan<- string) error {
+		tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
+		files <- tsPath
+
+		nquads, err := os.Create(tsPath)
+		if err != nil {
+			return err
+		}
+		defer nquads.Close()
+
+		// directly store the result
+		_, err = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
+		return err
+	}, &snapshot.ErrTriplestore)
+
+	// backup the sql database
+	og.GoErr(func(files chan<- string) error {
+		sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
+		files <- sqlPath
+
+		sql, err := os.Create(sqlPath)
+		if err != nil {
+			return err
+		}
+		defer sql.Close()
+
+		// directly store the result
+		return instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
+	}, &snapshot.ErrSQL)
+
+	// wait for the group!
+	snapshot.waitGroup(io, og)
+}
+
+// makeWhitebox runs the whitebox backup of the system.
+// The instance should be running during this step.
+func (snapshot *Snapshot) makeWhitebox(io stream.IOStream, instance Instance) {
+	og := opgroup.NewOpGroup[string](1)
+
+	// write pathbuilders
+	og.GoErr(func(files chan<- string) error {
+
+		pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
+		files <- pbPath
+
+		// create the directory!
+		if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
+			return err
+		}
+
+		// put in all the pathbuilders
+		return instance.ExportPathbuilders(pbPath)
+	}, &snapshot.ErrPathbuilder)
+
+	// wait for the group!
+	snapshot.waitGroup(io, og)
+}
+
+// waitGroup waits for the
+func (snapshot *Snapshot) waitGroup(io stream.IOStream, og *opgroup.OpGroup[string]) {
+	// wait for the messages to return
+	for file := range og.Wait() {
+		// get the relative path to the root of the manifest.
+		// nothing *should* go wrong, but in case it does, use the original path.
+		path, err := filepath.Rel(snapshot.Description.Dest, file)
+		if err != nil {
+			path = file
+		}
+
+		// write it to the command line
+		// and also add it to the manifest
+		io.Printf("\033[2K\r%s", path)
+		snapshot.Manifest = append(snapshot.Manifest, path)
+	}
+	io.Println("")
+}
+
+// WriteReport writes out the report belonging to this snapshot.
+// It is a separate function, to allow writing it indepenently of the rest.
+func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
+	return logging.LogOperation(func() error {
+		reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
+		io.Println(reportPath)
+
+		// create the report file!
+		report, err := os.Create(reportPath)
+		if err != nil {
+			return err
+		}
+		defer report.Close()
+
+		// print the report into it!
+		_, err = report.WriteString(snapshot.String())
+		return err
+	}, io, "Writing snapshot report")
+}