config: Properly use yaml template

This commit is contained in:
Tom Wiesing 2023-02-25 20:26:54 +01:00
parent 945329a080
commit 30c25b8e2a
No known key found for this signature in database
9 changed files with 168 additions and 78 deletions

View file

@ -96,7 +96,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
// TODO: Should we read an existing configuration file?
wdcliPath := filepath.Join(root, bootstrap.Executable)
envPath := filepath.Join(root, bootstrap.ConfigFile)
cfgPath := filepath.Join(root, bootstrap.ConfigFile)
// setup a new template for the configuration file!
var tpl config.Template
@ -123,25 +123,16 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
{
if !fsx.IsFile(env, envPath) {
if !fsx.IsFile(env, cfgPath) {
// generate the configuration from the template
cfg := tpl.Generate()
// write out all the extra config files
if err := logging.LogOperation(func() error {
env, err := env.Create(envPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer env.Close()
return tpl.MarshalTo(env)
}, context.Stderr, context.Context, "Installing configuration file"); err != nil {
return errBootstrapWriteConfig.WithMessageF(err)
}
if err := logging.LogOperation(func() error {
context.Println(tpl.SelfOverridesFile)
if err := environment.WriteFile(
env,
tpl.SelfOverridesFile,
cfg.Paths.OverridesJSON,
bootstrap.DefaultOverridesJSON,
fs.ModePerm,
); err != nil {
@ -151,7 +142,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
context.Println(tpl.SelfResolverBlockFile)
if err := environment.WriteFile(
env,
tpl.SelfResolverBlockFile,
cfg.Paths.ResolverBlocks,
bootstrap.DefaultResolverBlockedTXT,
fs.ModePerm,
); err != nil {
@ -159,16 +150,42 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
return nil
}, context.Stderr, context.Context, "Creating additional config files"); err != nil {
}, context.Stderr, context.Context, "Creating custom config files"); err != nil {
return errBootstrapCreateFile.WithMessageF(err)
}
// Validate configuration file!
if err := cfg.Validate(env); err != nil {
return err
}
// and marshal it out!
if err := logging.LogOperation(func() error {
configYML, err := env.Create(cfgPath, environment.DefaultFilePerm)
if err != nil {
return err
}
defer configYML.Close()
bytes, err := config.Marshal(&cfg, nil)
if err != nil {
return err
}
{
_, err := configYML.Write(bytes)
return err
}
}, context.Stderr, context.Context, "Installing primary configuration file"); err != nil {
return errBootstrapWriteConfig.WithMessageF(err)
}
}
}
// re-read the configuration and print it!
logging.LogMessage(context.Stderr, context.Context, "Configuration is now complete")
f, err := env.Open(envPath)
f, err := env.Open(cfgPath)
if err != nil {
return errBootstrapOpenConfig.WithMessageF(err)
}
@ -182,7 +199,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
// Tell the user how to proceed
logging.LogMessage(context.Stderr, context.Context, "Bootstrap is complete")
context.Printf("Adjust the configuration file at %s\n", envPath)
context.Printf("Adjust the configuration file at %s\n", cfgPath)
context.Printf("Then make sure 'docker compose' is installed.\n")
context.Printf("Finally grab a GraphDB zipped source file and run:\n")
context.Printf("%s system_update /path/to/graphdb.zip\n", wdcliPath)

View file

@ -1,7 +1,7 @@
package cmd
import (
"bytes"
"fmt"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
@ -44,21 +44,23 @@ func (c cfgMigrate) Run(context wisski_distillery.Context) error {
// then marshal, and re-read
var cfg config.Config
{
var mconfig config.Config
var output bytes.Buffer
if err := legacy.Migrate(&mconfig, env, file); err != nil {
return err
}
if err := mconfig.Marshal(&output); err != nil {
return err
}
if err := cfg.Unmarshal(env, &output); err != nil {
return err
}
// migrate the legacy config
if err := legacy.Migrate(&cfg, env, file); err != nil {
return err
}
// do a final marshal
return cfg.Marshal(context.Stdout)
// validate it!
if err := cfg.Validate(env); err != nil {
return err
}
// marshal the config
bytes, err := config.Marshal(&cfg, nil)
if err != nil {
return err
}
// and print it!
fmt.Println(string(bytes))
return nil
}

9
go.mod
View file

@ -15,15 +15,16 @@ require (
github.com/julienschmidt/httprouter v1.3.0
github.com/pkg/errors v0.9.1
github.com/pquerna/otp v1.4.0
github.com/rs/zerolog v1.28.0
github.com/rs/zerolog v1.29.0
github.com/tdewolff/minify v2.3.6+incompatible
github.com/tkw1536/goprogram v0.2.4
github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292
github.com/yuin/goldmark v1.4.13
github.com/yuin/goldmark-meta v1.1.0
golang.org/x/crypto v0.3.0
golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
golang.org/x/sync v0.1.0
golang.org/x/term v0.3.0
golang.org/x/term v0.5.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/mysql v1.4.4
gorm.io/gorm v1.24.2
@ -41,7 +42,7 @@ require (
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/tdewolff/parse v2.3.4+incompatible // indirect
github.com/tdewolff/test v1.0.7 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/tools v0.4.0 // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
)

10
go.sum
View file

@ -52,6 +52,8 @@ github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -63,6 +65,8 @@ github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM=
github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/tkw1536/goprogram v0.2.4 h1:1l3+j8xjY3E3uf+ba3QRGWm09ucFCKrnNLq6g1Gq8YA=
github.com/tkw1536/goprogram v0.2.4/go.mod h1:3Ngcwy7jtsZ+pINc+JfLdf8TWbvthdSS2T6Vbg44Fy8=
github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292 h1:/OeSU0bywyN9C4AJ9TARXUKxCZFK0lkZUp0MoHXCa2k=
github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292/go.mod h1:R+8tKMAkSXC1+XGzxNUKx2DnPJqObycYeo4PKjWYkMg=
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
@ -72,6 +76,8 @@ golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 h1:fJwx88sMf5RXwDwziL0/Mn9Wqs+efMSo/RYcL+37W9c=
golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w=
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
@ -87,11 +93,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

View file

@ -9,6 +9,10 @@ import (
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/pools"
"github.com/tkw1536/pkglib/yamlx"
"gopkg.in/yaml.v3"
_ "embed"
)
// Config represents the configuration of a WissKI Distillery.
@ -41,12 +45,47 @@ type Config struct {
SessionSecret string `yaml:"session_secret" default:"" validate:"nonempty"`
// interval to trigger distillery cron tasks in
CronInterval time.Duration `env:"cron_interval" default:"10m" validate:"duration"`
CronInterval time.Duration `yaml:"cron_interval" default:"10m" validate:"duration"`
// ConfigPath is the path this configuration was loaded from (if any)
ConfigPath string `yaml:"-"`
}
//go:embed config.yml
var configBytes []byte
// Marshal marshals this configuration in nicely formatted form.
// Where possible, this will provided yaml comments.
//
// Previous may optionally provide the bytes of a previous configuration file to replace settings in.
// The previous yaml file must be a valid configuration yaml, meaning all fields should be set.
// When previous is of length 0, the default configuration yaml will be used instead.
func Marshal(config *Config, previous []byte) ([]byte, error) {
if len(previous) == 0 {
previous = configBytes
}
// load the template yaml
template := new(yaml.Node)
if err := yaml.Unmarshal(previous, template); err != nil {
return nil, err
}
// load the config yaml
cfg, err := yamlx.Marshal(config)
if err != nil {
return nil, err
}
// transplant the configuration yaml into the template
if err := yamlx.Transplant(template, cfg); err != nil {
return nil, err
}
// marshal it again as a set of bytes
return yaml.Marshal(template)
}
// CSRFSecret return the csrfSecret derived from the session secret
func (config *Config) CSRFSecret() []byte {
// take the hash of the secret

View file

@ -1,17 +1,19 @@
paths:
# Several files are required to manage the system
# On top of this all real-system space will be created under this directory.
root: ${DEPLOY_ROOT}
# A WissKI Distillery needs to store a lot of data on disk.
# This setting defines a root folder all of these will be placed in.
# On top of this all real-system space will be created under this directory
root: ""
# You can override individual URLS in the homepage.
# Do this by adding URLs (without trailing '/'s) into a JSON file.
# This is the path to that file.
overrides: ${SELF_OVERRIDES_FILE}
# Individual paths on the root domain can be overwritten.
# This can be achieved by adding URLs (without trailing '/'s) into a JSON file.
# This setting defines the path to that file.
overrides: ""
# You can block specific prefixes within Triplestore from showing up in the global resolver.
# Do this by adding one prefix per line in this file.
# Lines starting with '#' and blank lines are ignored.
blocks: ${SELF_RESOLVER_BLOCK_FILE}
# This setting defines the path to that file.
blocks: ""
http:
# Each created Drupal Instance corresponds to a single domain name.
@ -30,7 +32,8 @@ http:
# By default, the default domain redirects to the distillery repository.
# If you want to change this, set an alternate domain name here.
home: ""
theme:
home: ""
docker:
# The name of the (global) docker network to run the distillery services in.

View file

@ -8,7 +8,7 @@ type DatabaseConfig struct {
// Prefix for new users and data setss
UserPrefix string `yaml:"user_prefix" default:"wisski-distillery-" validate:"slug"`
DataPrefix string `yaml:"fragment_prefix" default:"wisski-distillery-" validate:"slug"`
DataPrefix string `yaml:"data_prefix" default:"wisski-distillery-" validate:"slug"`
}
type SQLConfig struct {

View file

@ -21,7 +21,12 @@ func (config *Config) Unmarshal(env environment.Environment, src io.Reader) erro
}
}
// do the validator
// TODO: should this be done seperatly?
return config.Validate(env)
}
// Validate validates this configuration file and sets appropriate defaults
func (config *Config) Validate(env environment.Environment) error {
return validator.Validate(config, validators.New(env))
}

View file

@ -1,19 +1,13 @@
package config
import (
"bytes"
"io"
"path/filepath"
"reflect"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/bootstrap"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/hostname"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"gopkg.in/yaml.v3"
_ "embed"
)
// Template is a template for the configuration file
@ -88,28 +82,47 @@ func (tpl *Template) SetDefaults(env environment.Environment) (err error) {
return nil
}
//go:embed config_template.yml
var templateBytes []byte
// Generate generates a configuration file for this configuration
func (tpl Template) Generate() Config {
return Config{
Paths: PathsConfig{
Root: tpl.DeployRoot,
OverridesJSON: tpl.SelfOverridesFile,
ResolverBlocks: tpl.SelfResolverBlockFile,
},
HTTP: HTTPConfig{
PrimaryDomain: tpl.DefaultDomain,
ExtraDomains: []string{},
},
Docker: DockerConfig{
tpl.DockerNetworkName,
},
SQL: SQLConfig{
DatabaseConfig: DatabaseConfig{
AdminUsername: tpl.MysqlAdminUsername,
AdminPassword: tpl.MysqlAdminPassword,
// MarshalTo marshals this template into dst
func (tpl Template) MarshalTo(dst io.Writer) error {
tplVal := reflect.ValueOf(tpl)
tplType := reflect.TypeOf(tpl)
UserPrefix: "mysql-factory-",
DataPrefix: "mysql-factory-",
},
context := make(map[string]string, tplType.NumField())
for i := 0; i < tplType.NumField(); i++ {
field := tplType.Field(i)
Database: "distillery",
},
TS: TSConfig{
DatabaseConfig: DatabaseConfig{
AdminUsername: tpl.TriplestoreAdminUser,
AdminPassword: tpl.TriplestoreAdminPassword,
key := field.Tag.Get("env")
value := tplVal.FieldByName(field.Name).Interface()
UserPrefix: "graphdb-factory-",
DataPrefix: "graphdb-factory-",
},
},
MaxBackupAge: 30 * 24 * time.Hour, // 1 month
PasswordLength: 64,
bytes, err := yaml.Marshal(value)
if err != nil {
return err
}
context[key] = string(bytes)
PublicSSHPort: 2222,
SessionSecret: tpl.SessionSecret,
CronInterval: 10 * time.Minute,
}
// TODO: CONFIG: Update template writing
return unpack.WriteTemplate(dst, context, bytes.NewReader(templateBytes))
}