diff --git a/internal/dis/component/binder/binder.env b/internal/dis/component/binder/binder.env deleted file mode 100644 index 533fa65..0000000 --- a/internal/dis/component/binder/binder.env +++ /dev/null @@ -1 +0,0 @@ -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} \ No newline at end of file diff --git a/internal/dis/component/binder/binder.go b/internal/dis/component/binder/binder.go index 7545b0a..c61881e 100644 --- a/internal/dis/component/binder/binder.go +++ b/internal/dis/component/binder/binder.go @@ -2,13 +2,14 @@ package binder import ( "bytes" - "embed" "io" "path/filepath" "github.com/FAU-CDI/wisski-distillery/internal/dis/component" "github.com/tkw1536/pkglib/yamlx" "gopkg.in/yaml.v3" + + _ "embed" ) type Binder struct { @@ -66,13 +67,8 @@ func (binder *Binder) buildYML() ([]byte, error) { return yaml.Marshal(dockerCompose) } -//go:embed binder.env -var resources embed.FS - func (binder *Binder) Stack() component.StackWithResources { return component.MakeStack(binder, component.StackWithResources{ - Resources: resources, - EnvPath: "binder.env", ReadComposeFile: func() (io.Reader, error) { data, err := binder.buildYML() if err != nil { diff --git a/internal/dis/component/server/server.env b/internal/dis/component/server/server.env deleted file mode 100644 index e8d8e43..0000000 --- a/internal/dis/component/server/server.env +++ /dev/null @@ -1,11 +0,0 @@ -HOST_RULE=${HOST_RULE} - -CONFIG_PATH=${CONFIG_PATH} -DEPLOY_ROOT=${DEPLOY_ROOT} -SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} -SELF_RESOLVER_BLOCK_FILE=${SELF_RESOLVER_BLOCK_FILE} - -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} -HTTPS_ENABLED=${HTTPS_ENABLED} - -CUSTOM_ASSETS_PATH=${CUSTOM_ASSETS_PATH} \ No newline at end of file diff --git a/internal/dis/component/server/stack.go b/internal/dis/component/server/stack.go index addfe6d..d196a3e 100644 --- a/internal/dis/component/server/stack.go +++ b/internal/dis/component/server/stack.go @@ -15,14 +15,13 @@ func (control Server) Path() string { return filepath.Join(control.Still.Config.Paths.Root, "core", "dis") } -//go:embed all:server server.env +//go:embed all:server var resources embed.FS func (server *Server) Stack() component.StackWithResources { return component.MakeStack(server, component.StackWithResources{ Resources: resources, ContextPath: "server", - EnvPath: "server.env", EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": server.Config.Docker.Network(), diff --git a/internal/dis/component/solr/solr.env b/internal/dis/component/solr/solr.env deleted file mode 100644 index 131c6ab..0000000 --- a/internal/dis/component/solr/solr.env +++ /dev/null @@ -1 +0,0 @@ -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} diff --git a/internal/dis/component/solr/solr.go b/internal/dis/component/solr/solr.go index cb706a0..b2a6295 100644 --- a/internal/dis/component/solr/solr.go +++ b/internal/dis/component/solr/solr.go @@ -29,7 +29,6 @@ func (*Solr) Context(parent component.InstallationContext) component.Installatio } //go:embed all:solr -//go:embed solr.env var resources embed.FS func (solr *Solr) Stack() component.StackWithResources { @@ -37,7 +36,6 @@ func (solr *Solr) Stack() component.StackWithResources { Resources: resources, ContextPath: "solr", - EnvPath: "solr.env", EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": solr.Config.Docker.Network(), }, diff --git a/internal/dis/component/sql/sql.env b/internal/dis/component/sql/sql.env deleted file mode 100644 index 3089c4c..0000000 --- a/internal/dis/component/sql/sql.env +++ /dev/null @@ -1,2 +0,0 @@ -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} -HTTPS_ENABLED=${HTTPS_ENABLED} diff --git a/internal/dis/component/sql/sql.go b/internal/dis/component/sql/sql.go index 031e4bd..c27247e 100644 --- a/internal/dis/component/sql/sql.go +++ b/internal/dis/component/sql/sql.go @@ -37,7 +37,6 @@ func (*SQL) Context(parent component.InstallationContext) component.Installation } //go:embed all:sql -//go:embed sql.env var resources embed.FS func (sql *SQL) Stack() component.StackWithResources { @@ -45,7 +44,6 @@ func (sql *SQL) Stack() component.StackWithResources { Resources: resources, ContextPath: "sql", - EnvPath: "sql.env", EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": sql.Config.Docker.Network(), "HTTPS_ENABLED": sql.Config.HTTP.HTTPSEnabledEnv(), diff --git a/internal/dis/component/ssh2/ssh2.env b/internal/dis/component/ssh2/ssh2.env deleted file mode 100644 index 64c7d31..0000000 --- a/internal/dis/component/ssh2/ssh2.env +++ /dev/null @@ -1,8 +0,0 @@ -HOST_RULE=${HOST_RULE} - -CONFIG_PATH=${CONFIG_PATH} -DEPLOY_ROOT=${DEPLOY_ROOT} -SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} -SELF_RESOLVER_BLOCK_FILE=${SELF_RESOLVER_BLOCK_FILE} - -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} diff --git a/internal/dis/component/ssh2/stack.go b/internal/dis/component/ssh2/stack.go index d1601b0..7290919 100644 --- a/internal/dis/component/ssh2/stack.go +++ b/internal/dis/component/ssh2/stack.go @@ -12,14 +12,13 @@ func (ssh *SSH2) Path() string { return filepath.Join(ssh.Still.Config.Paths.Root, "core", "ssh2") } -//go:embed all:ssh2 ssh2.env +//go:embed all:ssh2 var resources embed.FS func (ssh *SSH2) Stack() component.StackWithResources { return component.MakeStack(ssh, component.StackWithResources{ Resources: resources, ContextPath: "ssh2", - EnvPath: "ssh2.env", EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": ssh.Config.Docker.Network(), diff --git a/internal/dis/component/stack.go b/internal/dis/component/stack.go index 64f6d2d..17f8425 100644 --- a/internal/dis/component/stack.go +++ b/internal/dis/component/stack.go @@ -176,10 +176,12 @@ type StackWithResources struct { // These all refer to paths within the Resource filesystem. Resources fs.FS - ContextPath string // the 'docker compose' stack context. Can, but does not have to, contain 'docker-compose.yml' + ContextPath string // the 'docker compose' stack context. Can, but does not have to, contain 'docker-compose.yml' + + // TODO: Make this nicer to replace variables ReadComposeFile func() (io.Reader, error) // read the 'docker-compose.yml' (if not contained in context) - EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate]. + // EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate]. If empty, use new syntax EnvContext map[string]string // context when instantiating the '.env' template CopyContextFiles []string // Files to copy from the installation context @@ -253,14 +255,10 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co // configure .env envDest := filepath.Join(is.Dir, ".env") - if is.EnvPath != "" && is.EnvContext != nil { + if is.EnvContext != nil { fmt.Fprintf(progress, "[config] %s\n", envDest) - if err := unpack.InstallTemplate( - envDest, - is.EnvContext, - is.EnvPath, - is.Resources, - ); err != nil { + + if err := writeEnvFile(envDest, is.TouchFilesPerm, is.EnvContext); err != nil { return err } } @@ -319,3 +317,22 @@ func (is StackWithResources) Install(ctx context.Context, progress io.Writer, co return nil } + +// writeEnvFile writes an environment file +func writeEnvFile(path string, perm fs.FileMode, variables map[string]string) error { + // create the environment file + file, err := umaskfree.Create(path, perm) + if err != nil { + return err + } + defer file.Close() + + // write the file! + _, err = compose.WriteEnvFile(file, variables) + if err != nil { + return err + } + + // and return nil + return nil +} diff --git a/internal/dis/component/triplestore/triplestore.env b/internal/dis/component/triplestore/triplestore.env deleted file mode 100644 index 131c6ab..0000000 --- a/internal/dis/component/triplestore/triplestore.env +++ /dev/null @@ -1 +0,0 @@ -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} diff --git a/internal/dis/component/triplestore/triplestore.go b/internal/dis/component/triplestore/triplestore.go index 7094449..921f916 100644 --- a/internal/dis/component/triplestore/triplestore.go +++ b/internal/dis/component/triplestore/triplestore.go @@ -32,7 +32,6 @@ func (Triplestore) Context(parent component.InstallationContext) component.Insta } //go:embed all:triplestore -//go:embed triplestore.env var resources embed.FS func (ts *Triplestore) Stack() component.StackWithResources { @@ -42,7 +41,6 @@ func (ts *Triplestore) Stack() component.StackWithResources { CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant? - EnvPath: "triplestore.env", EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": ts.Config.Docker.Network(), }, diff --git a/internal/dis/component/web/web.env b/internal/dis/component/web/web.env deleted file mode 100644 index 33613c0..0000000 --- a/internal/dis/component/web/web.env +++ /dev/null @@ -1,2 +0,0 @@ -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} -CERT_EMAIL=${CERT_EMAIL} \ No newline at end of file diff --git a/internal/dis/component/web/web.go b/internal/dis/component/web/web.go index fd0ec84..15c8e71 100644 --- a/internal/dis/component/web/web.go +++ b/internal/dis/component/web/web.go @@ -2,11 +2,12 @@ package web import ( "bytes" - "embed" "io" "path/filepath" "github.com/FAU-CDI/wisski-distillery/internal/dis/component" + + _ "embed" ) // Web implements the ingress gateway for the distillery. @@ -28,9 +29,6 @@ func (*Web) Context(parent component.InstallationContext) component.Installation return parent } -//go:embed web.env -var webEnv embed.FS - //go:embed docker-compose-http.yml var dockerComposeHTTP []byte @@ -39,8 +37,6 @@ var dockerComposeHTTPS []byte func (web *Web) Stack() component.StackWithResources { var stack component.StackWithResources - stack.Resources = webEnv - stack.EnvPath = "web.env" stack.EnvContext = map[string]string{ "DOCKER_NETWORK_NAME": web.Config.Docker.Network(), diff --git a/internal/wisski/ingredient/barrel/barrel.env b/internal/wisski/ingredient/barrel/barrel.env deleted file mode 100644 index 06b5189..0000000 --- a/internal/wisski/ingredient/barrel/barrel.env +++ /dev/null @@ -1,12 +0,0 @@ -DATA_PATH=${DATA_PATH} -RUNTIME_DIR=${RUNTIME_DIR} - -SLUG=${SLUG} -WISSKI_HOSTNAME=${HOSTNAME} -HOST_RULE=${HOST_RULE} -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} - -HTTPS_ENABLED=${HTTPS_ENABLED} - -BARREL_BASE_IMAGE=${BARREL_BASE_IMAGE} -OPCACHE_MODE=${OPCACHE_MODE} diff --git a/internal/wisski/ingredient/barrel/stack.go b/internal/wisski/ingredient/barrel/stack.go index d058efa..15f79c8 100644 --- a/internal/wisski/ingredient/barrel/stack.go +++ b/internal/wisski/ingredient/barrel/stack.go @@ -7,7 +7,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/dis/component" ) -//go:embed all:barrel barrel.env +//go:embed all:barrel var barrelResources embed.FS // Barrel returns a stack representing the running WissKI Instance @@ -19,15 +19,14 @@ func (barrel *Barrel) Stack() component.StackWithResources { Resources: barrelResources, ContextPath: filepath.Join("barrel"), - EnvPath: filepath.Join("barrel.env"), EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": barrel.Malt.Config.Docker.Network(), - "SLUG": barrel.Slug, - "HOST_RULE": barrel.HostRule(), - "HOSTNAME": barrel.Hostname(), - "HTTPS_ENABLED": barrel.Malt.Config.HTTP.HTTPSEnabledEnv(), + "SLUG": barrel.Slug, + "HOST_RULE": barrel.HostRule(), + "WISSKI_HOSTNAME": barrel.Hostname(), + "HTTPS_ENABLED": barrel.Malt.Config.HTTP.HTTPSEnabledEnv(), "DATA_PATH": filepath.Join(barrel.FilesystemBase, "data"), "RUNTIME_DIR": barrel.Malt.Config.Paths.RuntimeDir(), diff --git a/internal/wisski/ingredient/reserve/reserve.env b/internal/wisski/ingredient/reserve/reserve.env deleted file mode 100644 index 159fcd5..0000000 --- a/internal/wisski/ingredient/reserve/reserve.env +++ /dev/null @@ -1,5 +0,0 @@ -SLUG=${SLUG} -HOST_RULE=${HOST_RULE} - -DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} -HTTPS_ENABLED=${HTTPS_ENABLED} diff --git a/internal/wisski/ingredient/reserve/reserve.go b/internal/wisski/ingredient/reserve/reserve.go index 7999cd3..90e1488 100644 --- a/internal/wisski/ingredient/reserve/reserve.go +++ b/internal/wisski/ingredient/reserve/reserve.go @@ -14,7 +14,7 @@ type Reserve struct { ingredient.Base } -//go:embed all:reserve reserve.env +//go:embed all:reserve var reserveResources embed.FS // Stack returns a stack representing the reserve instance @@ -26,14 +26,12 @@ func (reserve *Reserve) Stack() component.StackWithResources { Resources: reserveResources, ContextPath: filepath.Join("reserve"), - EnvPath: filepath.Join("reserve.env"), EnvContext: map[string]string{ "DOCKER_NETWORK_NAME": reserve.Malt.Config.Docker.Network(), "SLUG": reserve.Slug, "HOST_RULE": reserve.HostRule(), - "HOSTNAME": reserve.Hostname(), "HTTPS_ENABLED": reserve.Malt.Config.HTTP.HTTPSEnabledEnv(), }, } diff --git a/pkg/compose/env.go b/pkg/compose/env.go new file mode 100644 index 0000000..2b8eea2 --- /dev/null +++ b/pkg/compose/env.go @@ -0,0 +1,142 @@ +package compose + +import ( + "fmt" + "io" + "strings" + + "github.com/tkw1536/pkglib/collection" +) + +const ( + EnvFileHeader = "# THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. \n\n" + + EnvEqualChar = '=' // assignment + EnvReplaceChar = '#' // replacement for invalid characters + EnvEscapeChar = '\\' // escaping + EnvQuoteChar = '"' // quoting +) + +type errInvalidName string + +func (ei errInvalidName) Error() string { + return fmt.Sprintf("invalid variable name: %q", string(ei)) +} + +// WriteEnvFile writes a .env file to io.Writer. +// Variables are written in consistent order. +// +// Variable names may only contain ascii letters, numbers or the character "_". +// Invalid variable names are an error. +// +// Variables values are escaped using EscapeEnvValue. +// +// count contains the number of bytes written to writer. +// In case of an error, partial content may already have been written to writer, as indicated by count. +func WriteEnvFile(writer io.Writer, env map[string]string) (count int, err error) { + var n int + + // write the header to the file + n, err = fmt.Fprint(writer, EnvFileHeader) + count += n + if err != nil { + return + } + + collection.IterateSorted(env, func(key, value string) { + // if we already had an error, break + if err != nil { + return + } + + // if we don't have a valid name, break + if !isValidVariable(key) { + err = errInvalidName(key) + return + } + + // write write key = EscapeEnvValue(value) followed by a new line + n, err = fmt.Fprintf(writer, "%s%s%s\n", key, string(EnvEqualChar), EscapeEnvValue(value)) + count += n + if err != nil { + return + } + }) + return +} + +// isValidVariable checks if name is a valid variable name. +func isValidVariable(name string) bool { + for _, r := range name { + if !(r == '_' || (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')) { + return false + } + } + return true +} + +// Escape escapes the given value to be written to an environment variable. +// If the value does not need escaping, it may return it unchanged. +// +// EscapeEnvValue allows ASCII characters from ' ' to '~' (inclusive) as well as '\t', '\r', '\n'. +// Other characters are automatically replaced by EnvReplaceChar. +func EscapeEnvValue(value string) (escaped string) { + // first check if we need to escape at all. + var changed bool + for _, r := range value { + if !isValidEnvChar(r) || r == '\n' || r == '\r' || r == '\t' || r == '$' || r == EnvEscapeChar || r == EnvQuoteChar { + changed = true + } + } + if !changed { + return value + } + + // make a new builder and make space for the original value + var builder strings.Builder + builder.Grow(len(value) + 2) + + // begin the quoting + builder.WriteRune(EnvQuoteChar) + + // iterate over it + for _, r := range value { + // if the character is invalid, it is replaced with an '_' + if !isValidEnvChar(r) { + builder.WriteRune(EnvReplaceChar) + continue + } + + switch r { + // custom escape for '\n', '\r', '\t' + case '\n': + builder.WriteRune(EnvEscapeChar) + builder.WriteRune('n') + case '\r': + builder.WriteRune(EnvEscapeChar) + builder.WriteRune('r') + case '\t': + builder.WriteRune(EnvEscapeChar) + builder.WriteRune('t') + + // standard escape for special characters + case '$', EnvEscapeChar, EnvQuoteChar: + builder.WriteRune(EnvEscapeChar) + fallthrough + + // that's it + default: + builder.WriteRune(r) + } + } + + // close the quote + builder.WriteRune(EnvQuoteChar) + + return builder.String() +} + +// isValidEnvChar checks if the rune r is allowed in environment variables. +func isValidEnvChar(r rune) bool { + return r == '\n' || r == '\r' || r == '\t' || (r >= ' ' && r <= '~') +} diff --git a/pkg/unpack/resource.go b/pkg/unpack/resource.go index 89f7fd8..a762a5c 100644 --- a/pkg/unpack/resource.go +++ b/pkg/unpack/resource.go @@ -10,7 +10,6 @@ import ( "github.com/tkw1536/pkglib/fsx/umaskfree" ) -var errExpectedFileButGotDirectory = errors.New("expected a file, but got a directory") var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a file") // InstallDir installs the directory at src within fsys to dst. diff --git a/pkg/unpack/template.go b/pkg/unpack/template.go index 4af64c7..ae4dba4 100644 --- a/pkg/unpack/template.go +++ b/pkg/unpack/template.go @@ -5,10 +5,8 @@ import ( "bufio" "fmt" "io" - "io/fs" "strings" - "github.com/tkw1536/pkglib/fsx/umaskfree" "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -196,38 +194,3 @@ parseloop: return nil } - -// InstallTemplate unpacks the resource located at src in fsys, then processes it as a template, and eventually writes it to dst. -// Any existing file is truncated and overwritten. -// -// See [WriteTemplate] for possible errors. -func InstallTemplate(dst string, context map[string]string, src string, fsys fs.FS) error { - - // open the srcFile - srcFile, err := fsys.Open(src) - if err != nil { - return err - } - defer srcFile.Close() - - // stat it - srcInfo, err := srcFile.Stat() - if err != nil { - return err - } - - // check if it is a directory - if srcInfo.IsDir() { - return errExpectedFileButGotDirectory - } - - // open the destination file - file, err := umaskfree.Create(dst, srcInfo.Mode()) - if err != nil { - return err - } - defer file.Close() - - // write the file! - return WriteTemplate(file, context, srcFile) -}