Move to yaml-based configuration

This commit updates the configuration to be yaml-based and updates the
configuration to read in a yaml file.

internal/config/legacy/envreader/envreader.go

@@ -0,0 +1,121 @@
+// Package envreader provides Scanner.
+// It is deprecated and will be removed in a future release.
+package envreader
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// Scanner is a scanner for environment files.
+// To create a new scanner use [NewScanner].
+//
+// It scans through a reader and reads environment variables from it.
+// Reads may be internally buffered.
+//
+// An environment variable is of the form:
+//
+//	KEY=VALUE
+//
+// on a separate line.
+// Keys and values are case-sensitive and may contain anything except for newline characters.
+// Spaces around key and value are trimmed using [strings.TrimSpace].
+// Keys may not contain an '='.
+// Lines not containing a '=' (e.g. blank lines) and those starting with '#' and '//' are ignored.
+//
+// To advance the scanner to the next key, value pair use [Scan].
+// To get the current (key, value) pair, use [Data].
+//
+// A typical use-case of a scanner is as follows:
+//
+//	scanner := NewScanner(r)
+//	for scanner.Scan() {
+//	    // process any data ....
+//	    fmt.Println(scanner.Data())
+//	}
+//	if err := scanner.Err(); err != nil {
+//	  	// handle errors
+//	}
+//
+// For the common use case of reading a set of distinct keys from a file see [ReadAll].
+type Scanner struct {
+	s *bufio.Scanner
+
+	// current key and value
+	key   string
+	value string
+}
+
+// NewScanner creates a new scanner from the underlying Reader
+func NewScanner(r io.Reader) *Scanner {
+	return &Scanner{
+		s: bufio.NewScanner(r),
+	}
+}
+
+// Scanner advances the scanner until the next KEY=VALUE pair.
+//
+// If there are no more values left (e.g. the underlying reader returned io.EOF)
+// or when an unexpected error occured, returns false.
+//
+// A caller should always check Err() to see if there was an error.
+func (scanner *Scanner) Scan() bool {
+	var found bool
+	for scanner.s.Scan() {
+		// check that we don't have an empty or comment only line
+		tokens := strings.TrimSpace(scanner.s.Text())
+		if len(tokens) == 0 || tokens[0] == '#' || strings.HasPrefix(tokens, "//") {
+			continue
+		}
+
+		// check that we have a 'key=value' pair
+		scanner.key, scanner.value, found = strings.Cut(tokens, "=")
+		if !found {
+			continue
+		}
+
+		// got a key = value
+		scanner.key = strings.TrimSpace(scanner.key)
+		scanner.value = strings.TrimSpace(scanner.value)
+		return true
+	}
+
+	// nothing found
+	scanner.key = ""
+	scanner.value = ""
+	return false
+}
+
+// Data reads the current value from the scanner.
+// When Scan() has not been called, or returned false, returns two empty strings.
+func (scanner Scanner) Data() (key, value string) {
+	return scanner.key, scanner.value
+}
+
+// Err returns any error that occured on the underlying read.
+//
+// When no error occured, or the underlying read is io.EOF, returns nil.
+func (scanner Scanner) Err() error {
+	return scanner.s.Err()
+}
+
+// ReadAll creates a new [Scanner], and then reads all key/value pairs from r.
+// If a key occurs more than once, only the last value is set in the returned map.
+func ReadAll(r io.Reader) (values map[string]string, err error) {
+	// TODO: This is no longer used
+	scanner := NewScanner(r)
+
+	// read and store all values
+	values = make(map[string]string)
+	for scanner.Scan() {
+		key, value := scanner.Data()
+		values[key] = value
+	}
+
+	// check if there was an error!
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return values, nil
+}

internal/config/legacy/envreader/envreader_test.go

@@ -0,0 +1,42 @@
+// Package envreader
+package envreader
+
+import (
+	"fmt"
+	"strings"
+)
+
+func ExampleNewScanner() {
+	scanner := NewScanner(strings.NewReader(`
+lines without an equal sign are ignored
+
+// this line is a comment, even with an = sign
+KEY=VALUE
+
+# this is also a comment =
+spaces in keys = spaces in values
+multiple=equal=signs
+CaSe = SenSitiVe
+empty value=
+=empty key
+`))
+
+	for scanner.Scan() {
+		key, value := scanner.Data()
+		fmt.Printf("%q %q\n", key, value)
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Println(scanner.Err())
+	} else {
+		fmt.Println("no error")
+	}
+
+	// Output: "KEY" "VALUE"
+	// "spaces in keys" "spaces in values"
+	// "multiple" "equal=signs"
+	// "CaSe" "SenSitiVe"
+	// "empty value" ""
+	// "" "empty key"
+	// no error
+}

internal/config/legacy/legacy.go

@@ -0,0 +1,147 @@
+// Package legacy provides support for reading legacy configuration.
+// It is deprecated and will be removed in a future release.
+package legacy
+
+import (
+	"io"
+	"net/url"
+	"reflect"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/config"
+	"github.com/FAU-CDI/wisski-distillery/internal/config/legacy/envreader"
+	"github.com/FAU-CDI/wisski-distillery/internal/config/legacy/stringparser"
+	"github.com/FAU-CDI/wisski-distillery/internal/config/validators"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/pkg/errors"
+)
+
+// Migrate parses a configuration from an old configuration.
+func Migrate(config *config.Config, env environment.Environment, src io.Reader) error {
+	var legacy Legacy
+	if err := legacy.Unmarshal(env, src); err != nil {
+		return nil
+	}
+	return legacy.Migrate(config)
+}
+
+// Legacy represents a legacy configuration file.
+//
+// NOTE(twiesing): This will be deprecated soon.
+type Legacy struct {
+	DeployRoot string `env:"DEPLOY_ROOT" default:"/var/www/deploy" parser:"abspath"`
+
+	DefaultDomain string `env:"DEFAULT_DOMAIN" default:"localhost.kwarc.info" parser:"domain"`
+
+	SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"https://github.com/FAU-CDI/wisski-distillery" parser:"https_url"`
+
+	SelfExtraDomains []string `env:"SELF_EXTRA_DOMAINS" default:"" parser:"domains"`
+
+	SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" parser:"file"`
+
+	SelfResolverBlockFile string `env:"SELF_RESOLVER_BLOCK_FILE" default:"" parser:"file"`
+
+	CertbotEmail string `env:"CERTBOT_EMAIL" default:"" parser:"email"`
+
+	MaxBackupAge int `env:"MAX_BACKUP_AGE" default:"" parser:"number"`
+
+	MysqlUserPrefix     string `env:"MYSQL_USER_PREFIX" default:"mysql-factory-" parser:"slug"`
+	MysqlDatabasePrefix string `env:"MYSQL_DATABASE_PREFIX" default:"mysql-factory-" parser:"slug"`
+	GraphDBUserPrefix   string `env:"GRAPHDB_USER_PREFIX" default:"mysql-factory-" parser:"slug"`
+	GraphDBRepoPrefix   string `env:"GRAPHDB_REPO_PREFIX" default:"mysql-factory-" parser:"slug"`
+
+	DistilleryDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" parser:"slug"`
+
+	PasswordLength int `env:"PASSWORD_LENGTH" default:"64" parser:"number"`
+
+	PublicSSHPort uint16 `env:"SSH_PORT" default:"2222" parser:"port"`
+
+	TriplestoreAdminUser     string `env:"GRAPHDB_ADMIN_USER" default:"admin" parser:"nonempty"`
+	TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD" default:"" parser:"nonempty"`
+
+	MysqlAdminUser     string `env:"MYSQL_ADMIN_USER" default:"admin" parser:"nonempty"`
+	MysqlAdminPassword string `env:"MYSQL_ADMIN_PASSWORD" default:"" parser:"nonempty"`
+
+	SessionSecret string `env:"SESSION_SECRET" default:"" parser:"nonempty"`
+
+	// name of docker network to use
+	DockerNetworkName string        `env:"DOCKER_NETWORK_NAME" default:"distillery" parser:"nonempty"`
+	CronInterval      time.Duration `env:"CRON_INTERVAL" default:"10m" parser:"duration"`
+}
+
+// Migrate migrates this LegacyConfig into a new configuration.
+func (legacy *Legacy) Migrate(cfg *config.Config) error {
+	cfg.Paths.Root = legacy.DeployRoot
+	cfg.HTTP.PrimaryDomain = legacy.DefaultDomain
+	cfg.Theme.SelfRedirect = (*validators.URL)(legacy.SelfRedirect)
+	cfg.HTTP.ExtraDomains = legacy.SelfExtraDomains
+	cfg.Paths.OverridesJSON = legacy.SelfOverridesFile
+	cfg.Paths.ResolverBlocks = legacy.SelfResolverBlockFile
+	cfg.HTTP.CertbotEmail = legacy.CertbotEmail
+	cfg.MaxBackupAge = time.Duration(legacy.MaxBackupAge) * 24 * time.Hour
+	cfg.SQL.UserPrefix = legacy.MysqlUserPrefix
+	cfg.SQL.DataPrefix = legacy.MysqlDatabasePrefix
+	cfg.TS.UserPrefix = legacy.GraphDBUserPrefix
+	cfg.TS.DataPrefix = legacy.GraphDBRepoPrefix
+	cfg.SQL.Database = legacy.DistilleryDatabase
+	cfg.PasswordLength = legacy.PasswordLength
+	cfg.PublicSSHPort = legacy.PublicSSHPort
+	cfg.TS.AdminUsername = legacy.TriplestoreAdminUser
+	cfg.TS.AdminPassword = legacy.TriplestoreAdminPassword
+	cfg.SQL.AdminUsername = legacy.MysqlAdminUser
+	cfg.SQL.AdminPassword = legacy.MysqlAdminPassword
+	cfg.SessionSecret = legacy.SessionSecret
+	cfg.Docker.Network = legacy.DockerNetworkName
+	cfg.CronInterval = legacy.CronInterval
+	return nil
+}
+
+// Unmarshal opens a legacy configuration file.
+//
+// Data is read using the [envreader.ReadAll] method, see the appropriate documentation for the file format.
+//
+// The `env` and `parser` reflect tags of the [Config] struct determine the keys to read from, and the types to expect.
+// When a key is missing, it is set to the default value.
+//
+// See also [stringparser.Parse].
+func (config *Legacy) Unmarshal(env environment.Environment, src io.Reader) error {
+	// read all the values!
+	values, err := envreader.ReadAll(src)
+	if err != nil {
+		return err
+	}
+
+	vConfig := reflect.ValueOf(config).Elem()
+	tConfig := vConfig.Type()
+
+	// iterate over the types
+	numValues := tConfig.NumField()
+	for i := 0; i < numValues; i++ {
+		tField := tConfig.Field(i)
+		vField := vConfig.FieldByName(tField.Name)
+
+		tEnv := tField.Tag.Get("env")
+		tDefault := tField.Tag.Get("default")
+		tParser := tField.Tag.Get("parser")
+
+		// skip it if it isn't loaded!
+		if tEnv == "" {
+			continue
+		}
+
+		// read the value with a default
+		value, ok := values[tEnv]
+		if !ok || value == "" {
+			if tDefault != "" {
+				value = tDefault
+			}
+		}
+
+		// parse the value!
+		if err := stringparser.Parse(env, tParser, value, vField); err != nil {
+			return errors.Errorf("Legacy.Unmarshal: Setting %q, Parser %q: %s", tEnv, tParser, err)
+		}
+	}
+
+	return nil
+}

internal/config/legacy/stringparser/parse.go

@@ -0,0 +1,65 @@
+package stringparser
+
+import (
+	"reflect"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/pkg/errors"
+)
+
+var errUnknownParser = errors.New("unknown parser")
+
+// Parse parses the provided value with the parser.
+func Parse(env environment.Environment, name, value string, vField reflect.Value) error {
+
+	// use the validator
+	parser, ok := knownParsers[strings.ToLower(name)]
+	if parser == nil || !ok {
+		return errUnknownParser
+	}
+
+	// get the parsed value
+	checked, err := parser(env, value)
+	if err != nil {
+		return err
+	}
+
+	// set the value of the field
+	var errSet interface{}
+	func() {
+		defer func() {
+			errSet = recover()
+		}()
+		vField.Set(reflect.ValueOf(checked))
+	}()
+
+	// capture any error
+	if errSet != nil {
+		return errors.Errorf("set returned %v", name, errSet)
+	}
+
+	return nil
+}
+
+// knownParsers holds the known parsers
+var knownParsers map[string]Parser[any] = map[string]Parser[any]{
+	"abspath":   asGenericParser(ParseAbspath),
+	"domain":    asGenericParser(ParseValidDomain),
+	"domains":   asGenericParser(ParseValidDomains),
+	"duration":  asGenericParser(ParseDuration),
+	"number":    asGenericParser(ParseNumber),
+	"port":      asGenericParser(ParsePort),
+	"https_url": asGenericParser(ParseHttpsURL),
+	"slug":      asGenericParser(ParseSlug),
+	"file":      asGenericParser(ParseFile),
+	"email":     asGenericParser(ParseEmail),
+	"nonempty":  asGenericParser(ParseNonEmpty),
+}
+
+func asGenericParser[T any](parser Parser[T]) Parser[any] {
+	return func(env environment.Environment, s string) (value any, err error) {
+		value, err = parser(env, s)
+		return
+	}
+}

internal/config/legacy/stringparser/stringparser.go

@@ -0,0 +1,125 @@
+// Package stringparser provides Parser.
+// It is deprecated and will be removed in a future release.
+package stringparser
+
+import (
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/pkg/errors"
+)
+
+// Parser is used to read a value from a string and turn it into a golang value.
+// It is simultaniously used to validate particular setting.
+//
+// Parsers can be found in this package as functions called Parse*.
+// They are refered to by their name, e.g. ParseNonempty can be refered to by the name 'Nonempty'.
+// See [Parse].
+type Parser[T any] func(env environment.Environment, s string) (T, error)
+
+// ParseAbspath checks that s is an absolute path and returns it as-is
+func ParseAbspath(env environment.Environment, s string) (string, error) {
+	if !fsx.IsDirectory(env, s) {
+		return "", errors.Errorf("%q does not exist or is not a directory", s)
+	}
+	return s, nil
+}
+
+// ParseFile checks that s is a valid file and returns it as-is
+func ParseFile(env environment.Environment, s string) (string, error) {
+	if !fsx.IsFile(env, s) {
+		return "", errors.Errorf("%q does not exist or is not a regular file", s)
+	}
+	return s, nil
+}
+
+var errEmptyString = errors.New("value is empty")
+
+// ParseNonEmpty checks that s is a non-empty string and returns it as-is
+func ParseNonEmpty(env environment.Environment, s string) (string, error) {
+	if s == "" {
+		return "", errEmptyString
+	}
+	return s, nil
+}
+
+var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+// ParseValidDomain checks that s is a valid domain and returns it in lowercase
+func ParseValidDomain(env environment.Environment, s string) (string, error) {
+	if !regexpDomain.MatchString(s) {
+		return "", errors.Errorf("%q is not a valid domain", s)
+	}
+	return strings.ToLower(s), nil
+}
+
+// ParseValidDomains checks that s is a comma-seperated list of valid domains and returns them in lower case
+func ParseValidDomains(env environment.Environment, s string) ([]string, error) {
+	if len(s) == 0 {
+		return []string{}, nil
+	}
+	domains := strings.Split(strings.ToLower(s), ",")
+	for _, d := range domains {
+		if !regexpDomain.MatchString(d) {
+			return nil, errors.Errorf("%q is not a valid domain", d)
+		}
+	}
+	return domains, nil
+}
+
+// ParseNumber parses s as a decimal integer
+func ParseNumber(env environment.Environment, s string) (int, error) {
+	value, err := strconv.ParseInt(s, 10, 64)
+	return int(value), err
+}
+
+// ParsePort parses s as a port
+func ParsePort(env environment.Environment, s string) (uint16, error) {
+	value, err := strconv.ParseUint(s, 10, 16)
+	return uint16(value), err
+}
+
+// ParseHttpsURL parses a string into a url that starts with 'https://'
+func ParseHttpsURL(env environment.Environment, s string) (*url.URL, error) {
+	url, err := url.Parse(s)
+	if err != nil {
+		return nil, errors.Wrapf(err, "%q is not a valid URL", s)
+	}
+	if url.Scheme != "https" {
+		return nil, errors.Errorf("%q is not a valid https URL (%q)", s, url.Scheme)
+	}
+	return url, nil
+}
+
+var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+// ParseEmail checks that s represents an email, and then returns it as is.
+func ParseEmail(env environment.Environment, s string) (string, error) {
+	if s == "" { // no email provided
+		return "", nil
+	}
+	if !regexpEmail.MatchString(s) {
+		return "", errors.Errorf("%q is not a valid email", s)
+	}
+	return s, nil
+}
+
+var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
+
+// ParseSlug parses s as a slug and returns it as is.
+func ParseSlug(env environment.Environment, s string) (string, error) {
+	if !regexpSlug.MatchString(s) {
+		return "", errors.Errorf("%q is not a valid slug", s)
+	}
+	return s, nil
+}
+
+// ParseDuration parses a time.Duration
+func ParseDuration(env environment.Environment, s string) (time.Duration, error) {
+	return time.ParseDuration(s)
+}