Rename packages

This commit is contained in:
Tom Wiesing 2022-09-14 14:17:08 +02:00
parent 49b8760527
commit ef1243ea39
No known key found for this signature in database
47 changed files with 524 additions and 369 deletions

View file

@ -9,8 +9,6 @@ import (
type Dis struct {
component.ComponentBase
Executable string // path to the current executable
}
func (dis Dis) Name() string {
@ -37,12 +35,12 @@ func (dis Dis) Stack() component.Installable {
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
},
CopyContextFiles: []string{core.Executable},
CopyContextFiles: []string{dis.Config.CurrentExecutable()},
})
}
func (dis Dis) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: dis.Executable,
core.Executable: dis.Config.CurrentExecutable(),
}
}

View file

@ -1,46 +1,48 @@
// Package config implements reading and validating a WissKIDistillery configuration file.
// Package config provides the distillery configuration
package config
import (
"fmt"
"io"
"net/url"
"reflect"
"strings"
"github.com/pkg/errors"
)
// Config represents the configuration of a distillery instance
// Config represents the configuration of a WissKI Distillery.
//
// Config is read from a byte stream using [Unmarshal].
//
// Config contains many methods that do not require any interaction with any running components.
// Methods that require running components are instead store inside the [Distillery] or an appropriate [Component].
type Config struct {
// Several docker-compose files are created to manage global services and the system itself.
// On top of this all real-system space will be created under this directory.
DeployRoot string `env:"DEPLOY_ROOT" default:"/var/www/deploy" validator:"is_valid_abspath"`
DeployRoot string `env:"DEPLOY_ROOT" default:"/var/www/deploy" parser:"abspath"`
// Each created Drupal Instance corresponds to a single domain name.
// These domain names should either be a complete domain name or a sub-domain of a default domain.
// This setting configures the default domain-name to create subdomains of.
DefaultDomain string `env:"DEFAULT_DOMAIN" default:"localhost.kwarc.info" validator:"is_valid_domain"`
DefaultDomain string `env:"DEFAULT_DOMAIN" default:"localhost.kwarc.info" parser:"domain"`
// By default, the default domain redirects to the distillery repository.
// If you want to change this, set an alternate domain name here.
SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"" validator:"is_valid_https_url"`
SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"" parser:"https_url"`
// By default, only the 'self' domain above is caught.
// To catch additional domains, add them here (comma seperated)
SelfExtraDomains []string `env:"SELF_EXTRA_DOMAINS" default:"" validator:"is_valid_domains"`
SelfExtraDomains []string `env:"SELF_EXTRA_DOMAINS" default:"" parser:"domains"`
// You can override individual URLS in the homepage
// Do this by adding URLs (without trailing '/'s) into a JSON file
SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" validator:"is_valid_file"`
SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" parser:"file"`
// The system can support setting up certificate(s) automatically.
// It can be enabled by setting an email for certbot certificates.
// This email address can be configured here.
CertbotEmail string `env:"CERTBOT_EMAIL" default:"" validator:"is_valid_email"`
CertbotEmail string `env:"CERTBOT_EMAIL" default:"" parser:"email"`
// Maximum age for backup in days
MaxBackupAge int `env:"MAX_BACKUP_AGE" default:"" validator:"is_valid_number"`
MaxBackupAge int `env:"MAX_BACKUP_AGE" default:"" parser:"number"`
// Each Drupal instance requires a corresponding system user, database users and databases.
// These are also set by the appropriate domain name.
@ -48,36 +50,37 @@ type Config struct {
// The prefix to use can be configured here.
// When changing these please consider that no system user may exist that has the same name as a mysql user.
// This is a MariaDB restriction.
MysqlUserPrefix string `env:"MYSQL_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
MysqlDatabasePrefix string `env:"MYSQL_DATABASE_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
GraphDBUserPrefix string `env:"GRAPHDB_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
GraphDBRepoPrefix string `env:"GRAPHDB_REPO_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"`
MysqlUserPrefix string `env:"MYSQL_USER_PREFIX" default:"mysql-factory-" parser:"slug"`
MysqlDatabasePrefix string `env:"MYSQL_DATABASE_PREFIX" default:"mysql-factory-" parser:"slug"`
GraphDBUserPrefix string `env:"GRAPHDB_USER_PREFIX" default:"mysql-factory-" parser:"slug"`
GraphDBRepoPrefix string `env:"GRAPHDB_REPO_PREFIX" default:"mysql-factory-" parser:"slug"`
// In addition to the filesystem the WissKI distillery requires a single SQL table.
// It uses this database to store a list of installed things.
DistilleryBookkeepingDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" validator:"is_valid_slug"`
DistilleryBookkeepingTable string `env:"DISTILLERY_BOOKKEEPING_TABLE" default:"distillery" validator:"is_valid_slug"`
DistilleryBookkeepingDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" parser:"slug"`
DistilleryBookkeepingTable string `env:"DISTILLERY_BOOKKEEPING_TABLE" default:"distillery" parser:"slug"`
// Various components use password-based-authentication.
// These passwords are generated automatically.
// This variable can be used to determine their length.
PasswordLength int `env:"PASSWORD_LENGTH" default:"64" validator:"is_valid_number"`
PasswordLength int `env:"PASSWORD_LENGTH" default:"64" parser:"number"`
// A file to be used for global authorized_keys for the ssh server.
GlobalAuthorizedKeysFile string `env:"GLOBAL_AUTHORIZED_KEYS_FILE" default:"/var/www/deploy/authorized_keys" validator:"is_valid_file"`
GlobalAuthorizedKeysFile string `env:"GLOBAL_AUTHORIZED_KEYS_FILE" default:"/var/www/deploy/authorized_keys" parser:"file"`
// admin credentials for graphdb
TriplestoreAdminUser string `env:"GRAPHDB_ADMIN_USER" default:"admin" validator:"is_nonempty"`
TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD" default:"" validator:"is_nonempty"`
TriplestoreAdminUser string `env:"GRAPHDB_ADMIN_USER" default:"admin" parser:"nonempty"`
TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD" default:"" parser:"nonempty"`
// admin credentials for the Mysql database
MysqlAdminUser string `env:"MYSQL_ADMIN_USER" default:"admin" validator:"is_nonempty"`
MysqlAdminPassword string `env:"MYSQL_ADMIN_PASSWORD" default:"admin" validator:"is_nonempty"`
MysqlAdminUser string `env:"MYSQL_ADMIN_USER" default:"admin" parser:"nonempty"`
MysqlAdminPassword string `env:"MYSQL_ADMIN_PASSWORD" default:"admin" parser:"nonempty"`
// ConfigPath is the path this configuration was loaded from (if any)
ConfigPath string
}
// String serializes this configuration into a string
func (config Config) String() string {
values := &strings.Builder{}
@ -100,67 +103,3 @@ func (config Config) String() string {
return values.String()
}
func (config *Config) Unmarshal(src io.Reader) error {
// read all the values!
values, err := ReadAll(src)
if err != nil {
return err
}
vConfig := reflect.ValueOf(config).Elem()
tConfig := vConfig.Type()
// iterate over the types
numValues := tConfig.NumField()
for i := 0; i < numValues; i++ {
tField := tConfig.Field(i)
vField := vConfig.FieldByName(tField.Name)
env := tField.Tag.Get("env")
dflt := tField.Tag.Get("default")
validator := tField.Tag.Get("validator")
// skip it if it isn't loaded!
if env == "" {
continue
}
// read the value with a default
value, ok := values[env]
if !ok || value == "" {
if dflt == "" {
continue
}
value = dflt
}
// use the validator
vFunc, ok := knownValidators[validator]
if vFunc == nil || !ok {
return errors.Errorf("Unable to read %q refers to unknown validator %s", env, validator)
}
// get the parsed value
checked, err := vFunc(value)
if err != nil {
return errors.Wrapf(err, "Unable to read %q: Validator %s", env, validator)
}
// set the value of the field
var errSet interface{}
func() {
defer func() {
errSet = recover()
}()
vField.Set(reflect.ValueOf(checked))
}()
// capture any error
if errSet != nil {
return errors.Errorf("Unable to parse %q: validator %s returned %q", tField.Name, validator, errSet)
}
}
return nil
}

View file

@ -0,0 +1,33 @@
package config
import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
// ExecutablePath returns the path to the executable of this distillery.
func (cfg Config) ExecutablePath() string {
return filepath.Join(cfg.DeployRoot, core.Executable)
}
// UsingDistilleryExecutable checks if the current process is using the distillery executable
func (cfg Config) UsingDistilleryExecutable() bool {
exe, err := os.Executable()
if err != nil {
return false
}
return fsx.SameFile(exe, cfg.ExecutablePath())
}
// CurrentExecutable returns the path to the current executable being used.
// When it does not exist, falls back to the default executable.
func (cfg Config) CurrentExecutable() string {
exe, err := os.Executable()
if err != nil || !fsx.IsFile(exe) {
return cfg.ExecutablePath()
}
return exe
}

View file

@ -1,76 +0,0 @@
package config
import (
"bufio"
"io"
"strings"
)
// Scanner scans an io.Reader for a source file
type Scanner struct {
src *bufio.Scanner
key string
value string
}
func NewScanner(r io.Reader) *Scanner {
return &Scanner{
src: bufio.NewScanner(r),
}
}
// Scanner advances the scanner to the next variable
func (scanner *Scanner) Scan() bool {
for scanner.src.Scan() {
// check that we don't have an empty or comment only line
tokens := strings.TrimSpace(scanner.src.Text())
if len(tokens) == 0 || tokens[0] == '#' || strings.HasPrefix(tokens, "//") {
continue
}
// check that we have a 'key=value' pair
values := strings.SplitN(tokens, "=", 2)
if len(values) != 2 {
continue
}
// got a key = value
scanner.key = strings.TrimSpace(values[0])
scanner.value = strings.TrimSpace(values[1])
return true
}
scanner.key = ""
scanner.value = ""
return false
}
// Data reads the current value from the scanner.
// When Scan() has not been called, or returned false, returns two empty strings.
func (scanner Scanner) Data() (key, value string) {
return scanner.key, scanner.value
}
// Error returns an error (if any)
func (scanner Scanner) Error() error {
return scanner.src.Err()
}
// ReadAll reads all key-value pairs from r.
// If a key occurs more than once, a later occurance overwrites a previous one.
func ReadAll(r io.Reader) (values map[string]string, err error) {
scanner := NewScanner(r)
// read and store all values
values = make(map[string]string)
for scanner.Scan() {
key, value := scanner.Data()
values[key] = value
}
// check if there was an error!
if err := scanner.Error(); err != nil {
return nil, err
}
return values, nil
}

60
internal/config/read.go Normal file
View file

@ -0,0 +1,60 @@
package config
import (
"io"
"reflect"
"github.com/FAU-CDI/wisski-distillery/pkg/envreader"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
)
// Unmarshal updates this configuration from the provided [io.Reader].
//
// Data is read using the [envreader.ReadAll] method, see the appropriate documentation for the file format.
//
// The `env` and `parser` reflect tags of the [Config] struct determine the keys to read from, and the types to expect.
// When a key is missing, it is set to the default value.
//
// See also [stringparser.Parse].
func (config *Config) Unmarshal(src io.Reader) error {
// read all the values!
values, err := envreader.ReadAll(src)
if err != nil {
return err
}
vConfig := reflect.ValueOf(config).Elem()
tConfig := vConfig.Type()
// iterate over the types
numValues := tConfig.NumField()
for i := 0; i < numValues; i++ {
tField := tConfig.Field(i)
vField := vConfig.FieldByName(tField.Name)
env := tField.Tag.Get("env")
dflt := tField.Tag.Get("default")
parser := tField.Tag.Get("parser")
// skip it if it isn't loaded!
if env == "" {
continue
}
// read the value with a default
value, ok := values[env]
if !ok || value == "" {
if dflt == "" {
continue
}
value = dflt
}
// parse the value!
if err := stringparser.Parse(parser, value, vField); err != nil {
return err
}
}
return nil
}

View file

@ -1,105 +0,0 @@
package config
import (
"net/url"
"regexp"
"strconv"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
)
// Validator reads from the configuration file
type Validator func(s string) (interface{}, error)
var knownValidators map[string]Validator = map[string]Validator{
"is_valid_abspath": IsValidAbspath,
"is_valid_domain": IsValidDomain,
"is_valid_domains": IsValidDomains,
"is_valid_number": IsValidNumber,
"is_valid_https_url": IsValidHttpsURL,
"is_valid_slug": IsValidSlug,
"is_valid_file": IsValidFile,
"is_valid_email": IsValidEmail,
"is_nonempty": IsNonEmpty,
}
func IsValidAbspath(s string) (interface{}, error) {
if !fsx.IsDirectory(s) {
return nil, errors.Errorf("%q does not exist or is not a directory", s)
}
return s, nil
}
func IsValidFile(s string) (interface{}, error) {
if !fsx.IsFile(s) {
return nil, errors.Errorf("%q does not exist or is not a regular file", s)
}
return s, nil
}
func IsNonEmpty(s string) (interface{}, error) {
if s == "" {
return nil, errors.New("value is empty")
}
return s, nil
}
var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
func IsValidDomain(s string) (interface{}, error) {
if !regexpDomain.MatchString(s) {
return nil, errors.Errorf("%q is not a valid domain", s)
}
return s, nil
}
func IsValidDomains(s string) (interface{}, error) {
if len(s) == 0 {
return []string{}, nil
}
domains := strings.Split(s, ",")
for _, d := range domains {
if !regexpDomain.MatchString(d) {
return nil, errors.Errorf("%q is not a valid domain", d)
}
}
return domains, nil
}
func IsValidNumber(s string) (interface{}, error) {
value, err := strconv.ParseInt(s, 10, 64)
return int(value), err
}
func IsValidHttpsURL(s string) (interface{}, error) {
url, err := url.Parse(s)
if err != nil {
return nil, errors.Wrapf(err, "%q is not a valid URL", s)
}
if url.Scheme != "https" {
return nil, errors.Errorf("%q is not a valid https URL (%q)", s, url.Scheme)
}
return url, nil
}
var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
func IsValidEmail(s string) (interface{}, error) {
if s == "" { // no email provided
return "", nil
}
if !regexpEmail.MatchString(s) {
return nil, errors.Errorf("%q is not a valid email", s)
}
return s, nil
}
var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
func IsValidSlug(s string) (interface{}, error) {
if !regexpSlug.MatchString(s) {
return nil, errors.Errorf("%q is not a valid slug", s)
}
return s, nil
}

View file

@ -33,3 +33,7 @@ const AuthorizedKeys = "authorized_keys"
// DefaultAuthorizedKeys contains a template for a new 'global_authorized_keys' file
//go:embed bootstrap/global_authorized_keys
var DefaultAuthorizedKeys []byte
// PrefixConfig is the name for the global resolver prefix configuration.
// It should be found within the prefix component directory.
const PrefixConfig = "prefix.cfg"

View file

@ -1,63 +0,0 @@
package env
import (
"context"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
// Distillery represents an interface to the running distillery.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// Upstream holds information to connect to the various running
// distillery components.
//
// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
// But for now this will just hold upstream configuration.
Upstream Upstream
// components hold references to the various components of the distillery.
components
}
// Upstream are the upstream urls connecting to the various external components.
type Upstream struct {
SQL string
Triplestore string
}
// Context returns a new Context belonging to this distillery
func (dis *Distillery) Context() context.Context {
return context.Background()
}
// ExecutablePath returns the path to the executable of this distillery.
func (dis *Distillery) ExecutablePath() string {
return filepath.Join(dis.Config.DeployRoot, core.Executable)
}
// UsingDistilleryExecutable checks if the current process
func (dis *Distillery) UsingDistilleryExecutable() bool {
exe, err := os.Executable()
if err != nil {
return false
}
return fsx.SameFile(exe, dis.ExecutablePath())
}
// CurrentExecutable returns the path to the current executable being used.
// When it does not exist, falls back to the default executable.
func (dis *Distillery) CurrentExecutable() string {
exe, err := os.Executable()
if err != nil || !fsx.IsFile(exe) {
return dis.ExecutablePath()
}
return exe
}

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"encoding/json"

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"path/filepath"
@ -14,6 +14,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/internal/component/web"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
// components holds the various components of the distillery
@ -94,15 +95,12 @@ func (dis *Distillery) Self() *self.Self {
func (dis *Distillery) Resolver() *resolver.Resolver {
return makeComponent(dis, &dis.components.resolver, func(resolver *resolver.Resolver) {
resolver.ConfigName = "prefix.cfg" // TODO: Move into core?
resolver.Executable = dis.CurrentExecutable()
resolver.ConfigName = core.PrefixConfig
})
}
func (d *Distillery) Dis() *dis.Dis {
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
ddis.Executable = d.CurrentExecutable()
})
return makeComponent(d, &d.components.dis, nil)
}
func (dis *Distillery) SSH() *ssh.SSH {

View file

@ -0,0 +1,37 @@
package wisski
import (
"context"
"github.com/FAU-CDI/wisski-distillery/internal/config"
)
// Distillery represents a WissKI Distillery
//
// It is the main structure used to interact with different components.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// Upstream holds information to connect to the various running
// distillery components.
//
// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
// But for now this will just hold upstream configuration.
Upstream Upstream
// components hold references to the various components of the distillery.
components
}
// Upstream are the upstream urls connecting to the various external components.
type Upstream struct {
SQL string
Triplestore string
}
// Context returns a new Context belonging to this distillery
func (dis *Distillery) Context() context.Context {
return context.Background()
}

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"os"
@ -18,9 +18,9 @@ var errOpenConfig = exit.Error{
Message: "error loading configuration file: %s",
}
// NewDistillery creates a new distillery object from a set of parameters and requirements
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (env *Distillery, err error) {
env = &Distillery{
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
@ -28,8 +28,8 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
}
if flags.InternalInDocker {
env.Upstream.SQL = "sql:3306"
env.Upstream.Triplestore = "triplestore:7200"
dis.Upstream.SQL = "sql:3306"
dis.Upstream.Triplestore = "triplestore:7200"
}
// if we don't need to load the config, there is nothing to do
@ -54,9 +54,9 @@ func NewDistillery(params core.Params, flags core.Flags, req core.Requirements)
defer f.Close()
// unmarshal the config
env.Config = &config.Config{
dis.Config = &config.Config{
ConfigPath: cfg,
}
err = env.Config.Unmarshal(f)
err = dis.Config.Unmarshal(f)
return
}

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"bytes"

View file

@ -1,10 +1,10 @@
package env
package wisski
import (
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
"github.com/pkg/errors"
)
@ -37,7 +37,7 @@ var errInvalidSlug = errors.New("Not a valid slug")
func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
// make sure that the slug is valid!
if _, err := config.IsValidSlug(slug); err != nil {
if _, err := stringparser.ParseSlug(slug); err != nil {
return i, errInvalidSlug
}

View file

@ -1,4 +1,4 @@
package env
package wisski
import "path/filepath"

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"io"

View file

@ -1,4 +1,4 @@
package env
package wisski
import (
"encoding/json"