Remove embed package

This commit finally removes the embed package in favor of more specific
resource packages

cmd/blind_update.go

@@ -45,7 +45,7 @@ func (bu blindUpdate) Run(context wisski_distillery.Context) error {
 		}
 		context.EPrintf("Updating instance %s\n", instance.Slug)
 
-		code, err := instance.Shell(context.IOStream, "/utils/blind_update.sh")
+		code, err := instance.Shell(context.IOStream, "/runtime/blind_update.sh")
 		if err != nil {
 			return errBlindUpdateFailed.WithMessageF(instance.Slug, execx.ExecCommandError)
 		}

cmd/bootstrap.go

@@ -1,13 +1,13 @@
 package cmd
 
 import (
+	"bytes"
 	"io/fs"
 	"os"
 	"path/filepath"
 
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/FAU-CDI/wisski-distillery/embed"
 	cfg "github.com/FAU-CDI/wisski-distillery/internal/config"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/FAU-CDI/wisski-distillery/internal/hostname"
@@ -124,8 +124,12 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
 					return errBootstrapWriteConfig.WithMessageF(err)
 				}
 
-				if err := unpack.InstallTemplate(
-					envPath,
+				env, err := os.Create(envPath)
+				if err != nil {
+					return errBootstrapWriteConfig.WithMessageF(err)
+				}
+				if err := unpack.WriteTemplate(
+					env,
 					map[string]string{
 						"DEPLOY_ROOT":          root,
 						"DEFAULT_DOMAIN":       domain,
@@ -138,8 +142,7 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
 						"MYSQL_ADMIN_USER":     "admin",
 						"MYSQL_ADMIN_PASSWORD": password[64:],
 					},
-					filepath.Join("resources", "templates", "bootstrap", "env"),
-					embed.ResourceEmbed,
+					bytes.NewReader(core.ConfigFileTemplate),
 				); err != nil {
 					return errBootstrapWriteConfig.WithMessageF(err)
 				}
@@ -152,17 +155,19 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
 			if err := logging.LogOperation(func() error {
 
 				context.Println(overridesPath)
-				if err := unpack.InstallFile(
+				if err := os.WriteFile(
 					overridesPath,
-					fsx.OpenFS(filepath.Join("resources", "templates", "bootstrap", "overrides.json"), embed.ResourceEmbed),
+					core.DefaultOverridesJSON,
+					fs.ModePerm,
 				); err != nil {
 					return errBootstrapCreateFile.WithMessageF(err)
 				}
 
 				context.Println(authorizedKeysFile)
-				if err := unpack.InstallFile(
+				if err := os.WriteFile(
 					authorizedKeysFile,
-					fsx.OpenFS(filepath.Join("resources", "templates", "bootstrap", "global_authorized_keys"), embed.ResourceEmbed),
+					core.DefaultAuthorizedKeys,
+					fs.ModePerm,
 				); err != nil {
 					return errBootstrapCreateFile.WithMessageF(err)
 				}

cmd/cron.go

@@ -40,7 +40,7 @@ func (cr cron) Run(context wisski_distillery.Context) error {
 	// iterate over the instances and store the last value of error
 	for _, instance := range instances {
 		logging.LogOperation(func() error {
-			code, err := instance.Shell(context.IOStream, "/utils/cron.sh")
+			code, err := instance.Shell(context.IOStream, "/runtime/cron.sh")
 			if err != nil {
 				context.EPrintln(err)
 			}

cmd/system_update.go

@@ -2,12 +2,10 @@ package cmd
 
 import (
 	"os"
-	"path/filepath"
 
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/component"
 	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/FAU-CDI/wisski-distillery/embed"
 	"github.com/FAU-CDI/wisski-distillery/internal/execx"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
 	"github.com/FAU-CDI/wisski-distillery/internal/unpack"
@@ -144,7 +142,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
 	}
 
 	if err := logging.LogOperation(func() error {
-		return unpack.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), embed.ResourceEmbed, func(dst, src string) {
+		return unpack.InstallResource(dis.RuntimeDir(), "runtime", core.Runtime, func(dst, src string) {
 			context.Printf("[copy]  %s\n", dst)
 		})
 	}, context.IOStream, "Unpacking Runtime Components"); err != nil {

component/triplestore/database.go

@@ -11,13 +11,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/FAU-CDI/wisski-distillery/embed"
-	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
-	"github.com/FAU-CDI/wisski-distillery/internal/unpack"
 	"github.com/FAU-CDI/wisski-distillery/internal/wait"
 	"github.com/pkg/errors"
-	"github.com/tkw1536/goprogram/exit"
 	"github.com/tkw1536/goprogram/stream"
 )
 
@@ -99,70 +95,6 @@ func (ts Triplestore) Wait() error {
 	}, ts.PollInterval, ts.PollContext)
 }
 
-var errTripleStoreFailedRepository = exit.Error{
-	Message:  "Failed to create repository: %s",
-	ExitCode: exit.ExitGeneric,
-}
-
-func (ts Triplestore) Provision(name, domain, user, password string) error {
-	if err := ts.Wait(); err != nil {
-		return err
-	}
-
-	// prepare the create repo request
-	// TODO: Move this into a seperate file
-	createRepo, _, err := unpack.UnpackTemplate(
-		map[string]string{
-			"GRAPHDB_REPO":    name,
-			"INSTANCE_DOMAIN": domain,
-		},
-		fsx.OpenFS(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), embed.ResourceEmbed),
-	)
-	if err != nil {
-		return err
-	}
-
-	// do the create!
-	{
-		res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo, "config", "")
-		if err != nil {
-			return errTripleStoreFailedRepository.WithMessageF(err)
-		}
-		defer res.Body.Close()
-		if res.StatusCode != http.StatusCreated {
-			return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
-		}
-	}
-
-	// create the user and grant them access
-	{
-		res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
-			Password: password,
-			AppSettings: TriplestoreUserAppSettings{
-				DefaultInference:      true,
-				DefaultVisGraphSchema: true,
-				DefaultSameas:         true,
-				IgnoreSharedQueries:   false,
-				ExecuteCount:          true,
-			},
-			GrantedAuthorities: []string{
-				"ROLE_USER",
-				"READ_REPO_" + name,
-				"WRITE_REPO_" + name,
-			},
-		}, "", "")
-		if err != nil {
-			return errTripleStoreFailedRepository.WithMessageF(err)
-		}
-		defer res.Body.Close()
-		if res.StatusCode != http.StatusCreated {
-			return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
-		}
-	}
-
-	return nil
-}
-
 // TriplestorePurgeUser deletes the specified user from the triplestore
 func (ts Triplestore) PurgeUser(user string) error {
 	res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")

component/triplestore/provision.go

@@ -0,0 +1,75 @@
+package triplestore
+
+import (
+	"bytes"
+	"net/http"
+
+	_ "embed"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/unpack"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+var errTripleStoreFailedRepository = exit.Error{
+	Message:  "Failed to create repository: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+//go:embed create-repo.ttl
+var createRepoTTL []byte
+
+func (ts Triplestore) Provision(name, domain, user, password string) error {
+	if err := ts.Wait(); err != nil {
+		return err
+	}
+
+	// prepare the create repo request
+	var createRepo bytes.Buffer
+	err := unpack.WriteTemplate(&createRepo, map[string]string{
+		"GRAPHDB_REPO":    name,
+		"INSTANCE_DOMAIN": domain,
+	}, bytes.NewReader(createRepoTTL))
+	if err != nil {
+		return err
+	}
+
+	// do the create!
+	{
+		res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo.Bytes(), "config", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
+		}
+	}
+
+	// create the user and grant them access
+	{
+		res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
+			Password: password,
+			AppSettings: TriplestoreUserAppSettings{
+				DefaultInference:      true,
+				DefaultVisGraphSchema: true,
+				DefaultSameas:         true,
+				IgnoreSharedQueries:   false,
+				ExecuteCount:          true,
+			},
+			GrantedAuthorities: []string{
+				"ROLE_USER",
+				"READ_REPO_" + name,
+				"WRITE_REPO_" + name,
+			},
+		}, "", "")
+		if err != nil {
+			return errTripleStoreFailedRepository.WithMessageF(err)
+		}
+		defer res.Body.Close()
+		if res.StatusCode != http.StatusCreated {
+			return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
+		}
+	}
+
+	return nil
+}

core/bootstrap.go

@@ -0,0 +1,15 @@
+package core
+
+import _ "embed"
+
+// DefaultOverridesJSON contains a template for a new 'overrides.json' file
+//go:embed bootstrap/overrides.json
+var DefaultOverridesJSON []byte
+
+// DefaultAuthorizedKeys contains a template for a new 'global_authorized_keys' file
+//go:embed bootstrap/global_authorized_keys
+var DefaultAuthorizedKeys []byte
+
+// ConfigFileTemplate contains a template for a new configuration file
+//go:embed bootstrap/env
+var ConfigFileTemplate []byte

core/bootstrap/global_authorized_keys

@@ -0,0 +1,2 @@
+# This file contains authorized_keys files valid for every repository in the distillery
+# The syntax of this file is easy, one key per line, empty lines or those starting with '#' are ignored

core/runtime.go

@@ -0,0 +1,10 @@
+package core
+
+import (
+	"embed"
+	_ "embed"
+)
+
+// Runtime contains runtime resources to be installed into any instance
+//go:embed all:runtime
+var Runtime embed.FS

core/runtime/README

@@ -1,2 +1,2 @@
 Files in this folder are utility scripts to be used from within individual WissKI instances. 
-They are mounted under /utils/ and should be used with care. 
+They are mounted under runtime/ and should be used with care. 

embed/legacy.go

@@ -1,10 +0,0 @@
-// Package embed contains embedded resources
-package embed
-
-import (
-	"embed"
-)
-
-// ResourceEmbed contains all the resources required by the WissKI-Distillery package.
-//go:embed all:resources
-var ResourceEmbed embed.FS

embed/resources/.dockerignore

@@ -1 +0,0 @@
-data/

embed/resources/templates/bootstrap/global_authorized_keys

@@ -1,2 +0,0 @@
-# This file contains authorized_keys files valid for every repository in the distillery.
-# To add a key, add one file per line.

env/constants.go

@@ -1,33 +0,0 @@
-package env
-
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
-)
-
-// ExecutablePath returns the path to the executable of this distillery.
-func (dis *Distillery) ExecutablePath() string {
-	return filepath.Join(dis.Config.DeployRoot, core.Executable)
-}
-
-// UsingDistilleryExecutable checks if the current process
-func (dis *Distillery) UsingDistilleryExecutable() bool {
-	exe, err := os.Executable()
-	if err != nil {
-		return false
-	}
-	return fsx.SameFile(exe, dis.ExecutablePath())
-}
-
-// CurrentExecutable returns the path to the current executable being used.
-// When it does not exist, falls back to the default executable.
-func (dis *Distillery) CurrentExecutable() string {
-	exe, err := os.Executable()
-	if err != nil || !fsx.IsFile(exe) {
-		return dis.ExecutablePath()
-	}
-	return exe
-}

env/distillery.go

@@ -3,10 +3,11 @@ package env
 import (
 	"context"
 	"os"
+	"path/filepath"
 
 	"github.com/FAU-CDI/wisski-distillery/core"
 	"github.com/FAU-CDI/wisski-distillery/internal/config"
-	"github.com/tkw1536/goprogram/exit"
+	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 )
 
 // Distillery represents a running instance for the distillery
@@ -26,55 +27,26 @@ func (dis Distillery) Context() context.Context {
 	return context.Background()
 }
 
-var errNoConfigFile = exit.Error{
-	ExitCode: exit.ExitGeneralArguments,
-	Message:  "Configuration File does not exist",
+// ExecutablePath returns the path to the executable of this distillery.
+func (dis *Distillery) ExecutablePath() string {
+	return filepath.Join(dis.Config.DeployRoot, core.Executable)
 }
 
-var errOpenConfig = exit.Error{
-	ExitCode: exit.ExitGeneralArguments,
-	Message:  "error loading configuration file: %s",
-}
-
-// NewDistillery creates a new distillery object from a set of parameters and requirements
-func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (env *Distillery, err error) {
-	env = &Distillery{
-		Upstream: Upstream{
-			SQL:         "127.0.0.1:3306",
-			Triplestore: "127.0.0.1:7200",
-		},
-	}
-
-	if flags.InternalInDocker {
-		env.Upstream.SQL = "sql:3306"
-		env.Upstream.Triplestore = "triplestore:7200"
-	}
-
-	// if we don't need to load the config, there is nothing to do
-	if !req.NeedsDistillery {
-		return
-	}
-
-	// try to find the configuration file
-	cfg := flags.ConfigPath // command line flags first
-	if cfg == "" {
-		cfg = params.ConfigPath // then globally provided files
-	}
-	if cfg == "" {
-		return nil, errNoConfigFile
-	}
-
-	// open the config file!
-	f, err := os.Open(params.ConfigPath)
+// UsingDistilleryExecutable checks if the current process
+func (dis *Distillery) UsingDistilleryExecutable() bool {
+	exe, err := os.Executable()
 	if err != nil {
-		return nil, errOpenConfig.WithMessageF(err)
+		return false
 	}
-	defer f.Close()
-
-	// unmarshal the config
-	env.Config = &config.Config{
-		ConfigPath: cfg,
-	}
-	err = env.Config.Unmarshal(f)
-	return
+	return fsx.SameFile(exe, dis.ExecutablePath())
+}
+
+// CurrentExecutable returns the path to the current executable being used.
+// When it does not exist, falls back to the default executable.
+func (dis *Distillery) CurrentExecutable() string {
+	exe, err := os.Executable()
+	if err != nil || !fsx.IsFile(exe) {
+		return dis.ExecutablePath()
+	}
+	return exe
 }

env/init.go

@@ -0,0 +1,62 @@
+package env
+
+import (
+	"os"
+
+	"github.com/FAU-CDI/wisski-distillery/core"
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+var errNoConfigFile = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "Configuration File does not exist",
+}
+
+var errOpenConfig = exit.Error{
+	ExitCode: exit.ExitGeneralArguments,
+	Message:  "error loading configuration file: %s",
+}
+
+// NewDistillery creates a new distillery object from a set of parameters and requirements
+func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (env *Distillery, err error) {
+	env = &Distillery{
+		Upstream: Upstream{
+			SQL:         "127.0.0.1:3306",
+			Triplestore: "127.0.0.1:7200",
+		},
+	}
+
+	if flags.InternalInDocker {
+		env.Upstream.SQL = "sql:3306"
+		env.Upstream.Triplestore = "triplestore:7200"
+	}
+
+	// if we don't need to load the config, there is nothing to do
+	if !req.NeedsDistillery {
+		return
+	}
+
+	// try to find the configuration file
+	cfg := flags.ConfigPath // command line flags first
+	if cfg == "" {
+		cfg = params.ConfigPath // then globally provided files
+	}
+	if cfg == "" {
+		return nil, errNoConfigFile
+	}
+
+	// open the config file!
+	f, err := os.Open(params.ConfigPath)
+	if err != nil {
+		return nil, errOpenConfig.WithMessageF(err)
+	}
+	defer f.Close()
+
+	// unmarshal the config
+	env.Config = &config.Config{
+		ConfigPath: cfg,
+	}
+	err = env.Config.Unmarshal(f)
+	return
+}

env/instances.go

@@ -2,6 +2,7 @@ package env
 
 import (
 	"bytes"
+	"embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,7 +13,6 @@ import (
 	"strings"
 
 	"github.com/FAU-CDI/wisski-distillery/component"
-	"github.com/FAU-CDI/wisski-distillery/embed"
 	"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/alessio/shellescape"
@@ -228,16 +228,20 @@ func (instance Instance) URL() *url.URL {
 	return url
 }
 
+//go:embed all:instances/barrel instances/barrel.env
+var barrelResources embed.FS
+
 // Stack represents a stack representing this instance
 func (instance Instance) Stack() component.Installable {
 	return component.Installable{
 		Stack: component.Stack{
 			Dir: instance.FilesystemBase,
 		},
-		Resources:   embed.ResourceEmbed, // TODO: Move this over
-		ContextPath: filepath.Join("resources", "compose", "barrel"),
 
-		EnvPath: filepath.Join("resources", "templates", "docker-env", "barrel"),
+		Resources:   barrelResources,
+		ContextPath: filepath.Join("instances", "barrel"),
+		EnvPath:     filepath.Join("instances", "barrel.env"),
+
 		EnvContext: map[string]string{
 			"DATA_PATH": filepath.Join(instance.FilesystemBase, "data"),
 
@@ -247,7 +251,7 @@ func (instance Instance) Stack() component.Installable {
 			"LETSENCRYPT_HOST":  instance.dis.IfHttps(instance.Domain()),
 			"LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail),
 
-			"UTILS_DIR":                   instance.dis.RuntimeUtilsDir(),
+			"RUNTIME_DIR":                 instance.dis.RuntimeDir(),
 			"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
 		},
 
@@ -260,14 +264,19 @@ func (instance Instance) Stack() component.Installable {
 	}
 }
 
+//go:embed all:instances/reserve instances/reserve.env
+var reserveResources embed.FS
+
 func (instance Instance) ReserveStack() component.Installable {
 	return component.Installable{
 		Stack: component.Stack{
 			Dir: instance.FilesystemBase,
 		},
-		ContextPath: filepath.Join("resources", "compose", "reserve"),
 
-		EnvPath: filepath.Join("resources", "templates", "docker-env", "reserve"),
+		Resources:   reserveResources,
+		ContextPath: filepath.Join("instances", "reserve"),
+		EnvPath:     filepath.Join("instances", "reserve.env"),
+
 		EnvContext: map[string]string{
 			"VIRTUAL_HOST": instance.Domain(),
 

env/instances/barrel.env

@@ -1,5 +1,5 @@
 DATA_PATH=${DATA_PATH}
-UTILS_DIR=${UTILS_DIR}
+RUNTIME_DIR=${RUNTIME_DIR}
 
 SLUG=${SLUG}
 VIRTUAL_HOST=${VIRTUAL_HOST}

env/instances/barrel/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       - ${DATA_PATH}/.composer:/var/www/.composer
       - ${DATA_PATH}/data:/var/www/data
       - ${DATA_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
-      - ${UTILS_DIR}:/utils:ro
+      - ${RUNTIME_DIR}:/runtime:ro
 
 networks:
   default:

env/runtime.go

@@ -6,8 +6,3 @@ import "path/filepath"
 func (dis Distillery) RuntimeDir() string {
 	return filepath.Join(dis.Config.DeployRoot, "runtime")
 }
-
-// RuntimeUtilsDir returns the path to the runtime utility dir
-func (dis Distillery) RuntimeUtilsDir() string {
-	return filepath.Join(dis.Config.DeployRoot, "runtime", "utils")
-}