diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 8670121..1ec1bb2 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -96,7 +96,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error { // TODO: Should we read an existing configuration file? wdcliPath := filepath.Join(root, bootstrap.Executable) - envPath := filepath.Join(root, bootstrap.ConfigFile) + cfgPath := filepath.Join(root, bootstrap.ConfigFile) // setup a new template for the configuration file! var tpl config.Template @@ -123,25 +123,16 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error { } { - if !fsx.IsFile(env, envPath) { + if !fsx.IsFile(env, cfgPath) { + // generate the configuration from the template + cfg := tpl.Generate() + + // write out all the extra config files if err := logging.LogOperation(func() error { - env, err := env.Create(envPath, environment.DefaultFilePerm) - if err != nil { - return err - } - defer env.Close() - - return tpl.MarshalTo(env) - }, context.Stderr, context.Context, "Installing configuration file"); err != nil { - return errBootstrapWriteConfig.WithMessageF(err) - } - - if err := logging.LogOperation(func() error { - context.Println(tpl.SelfOverridesFile) if err := environment.WriteFile( env, - tpl.SelfOverridesFile, + cfg.Paths.OverridesJSON, bootstrap.DefaultOverridesJSON, fs.ModePerm, ); err != nil { @@ -151,7 +142,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error { context.Println(tpl.SelfResolverBlockFile) if err := environment.WriteFile( env, - tpl.SelfResolverBlockFile, + cfg.Paths.ResolverBlocks, bootstrap.DefaultResolverBlockedTXT, fs.ModePerm, ); err != nil { @@ -159,16 +150,42 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error { } return nil - }, context.Stderr, context.Context, "Creating additional config files"); err != nil { + }, context.Stderr, context.Context, "Creating custom config files"); err != nil { return errBootstrapCreateFile.WithMessageF(err) } + + // Validate configuration file! + if err := cfg.Validate(env); err != nil { + return err + } + + // and marshal it out! + if err := logging.LogOperation(func() error { + configYML, err := env.Create(cfgPath, environment.DefaultFilePerm) + if err != nil { + return err + } + defer configYML.Close() + + bytes, err := config.Marshal(&cfg, nil) + if err != nil { + return err + } + + { + _, err := configYML.Write(bytes) + return err + } + }, context.Stderr, context.Context, "Installing primary configuration file"); err != nil { + return errBootstrapWriteConfig.WithMessageF(err) + } } } // re-read the configuration and print it! logging.LogMessage(context.Stderr, context.Context, "Configuration is now complete") - f, err := env.Open(envPath) + f, err := env.Open(cfgPath) if err != nil { return errBootstrapOpenConfig.WithMessageF(err) } @@ -182,7 +199,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error { // Tell the user how to proceed logging.LogMessage(context.Stderr, context.Context, "Bootstrap is complete") - context.Printf("Adjust the configuration file at %s\n", envPath) + context.Printf("Adjust the configuration file at %s\n", cfgPath) context.Printf("Then make sure 'docker compose' is installed.\n") context.Printf("Finally grab a GraphDB zipped source file and run:\n") context.Printf("%s system_update /path/to/graphdb.zip\n", wdcliPath) diff --git a/cmd/config_migrate.go b/cmd/config_migrate.go index 481a1a4..2beb339 100644 --- a/cmd/config_migrate.go +++ b/cmd/config_migrate.go @@ -1,7 +1,7 @@ package cmd import ( - "bytes" + "fmt" wisski_distillery "github.com/FAU-CDI/wisski-distillery" "github.com/FAU-CDI/wisski-distillery/internal/cli" @@ -44,21 +44,23 @@ func (c cfgMigrate) Run(context wisski_distillery.Context) error { // then marshal, and re-read var cfg config.Config - { - var mconfig config.Config - var output bytes.Buffer - - if err := legacy.Migrate(&mconfig, env, file); err != nil { - return err - } - if err := mconfig.Marshal(&output); err != nil { - return err - } - if err := cfg.Unmarshal(env, &output); err != nil { - return err - } + // migrate the legacy config + if err := legacy.Migrate(&cfg, env, file); err != nil { + return err } - // do a final marshal - return cfg.Marshal(context.Stdout) + // validate it! + if err := cfg.Validate(env); err != nil { + return err + } + + // marshal the config + bytes, err := config.Marshal(&cfg, nil) + if err != nil { + return err + } + + // and print it! + fmt.Println(string(bytes)) + return nil } diff --git a/go.mod b/go.mod index a616f0c..55a5c54 100644 --- a/go.mod +++ b/go.mod @@ -15,15 +15,16 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 - github.com/rs/zerolog v1.28.0 + github.com/rs/zerolog v1.29.0 github.com/tdewolff/minify v2.3.6+incompatible github.com/tkw1536/goprogram v0.2.4 + github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292 github.com/yuin/goldmark v1.4.13 github.com/yuin/goldmark-meta v1.1.0 golang.org/x/crypto v0.3.0 - golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 + golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb golang.org/x/sync v0.1.0 - golang.org/x/term v0.3.0 + golang.org/x/term v0.5.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.4.4 gorm.io/gorm v1.24.2 @@ -41,7 +42,7 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/tdewolff/parse v2.3.4+incompatible // indirect github.com/tdewolff/test v1.0.7 // indirect - golang.org/x/sys v0.3.0 // indirect + golang.org/x/sys v0.5.0 // indirect golang.org/x/tools v0.4.0 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect ) diff --git a/go.sum b/go.sum index 6d733be..e17c8db 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -63,6 +65,8 @@ github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM= github.com/tdewolff/test v1.0.7/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tkw1536/goprogram v0.2.4 h1:1l3+j8xjY3E3uf+ba3QRGWm09ucFCKrnNLq6g1Gq8YA= github.com/tkw1536/goprogram v0.2.4/go.mod h1:3Ngcwy7jtsZ+pINc+JfLdf8TWbvthdSS2T6Vbg44Fy8= +github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292 h1:/OeSU0bywyN9C4AJ9TARXUKxCZFK0lkZUp0MoHXCa2k= +github.com/tkw1536/pkglib v0.0.0-20230225192547-93a1aa42a292/go.mod h1:R+8tKMAkSXC1+XGzxNUKx2DnPJqObycYeo4PKjWYkMg= github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc= @@ -72,6 +76,8 @@ golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3 h1:fJwx88sMf5RXwDwziL0/Mn9Wqs+efMSo/RYcL+37W9c= golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= @@ -87,11 +93,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/config/config.go b/internal/config/config.go index 03d4e4e..a3cd191 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,10 @@ import ( "time" "github.com/FAU-CDI/wisski-distillery/pkg/pools" + "github.com/tkw1536/pkglib/yamlx" + "gopkg.in/yaml.v3" + + _ "embed" ) // Config represents the configuration of a WissKI Distillery. @@ -41,12 +45,47 @@ type Config struct { SessionSecret string `yaml:"session_secret" default:"" validate:"nonempty"` // interval to trigger distillery cron tasks in - CronInterval time.Duration `env:"cron_interval" default:"10m" validate:"duration"` + CronInterval time.Duration `yaml:"cron_interval" default:"10m" validate:"duration"` // ConfigPath is the path this configuration was loaded from (if any) ConfigPath string `yaml:"-"` } +//go:embed config.yml +var configBytes []byte + +// Marshal marshals this configuration in nicely formatted form. +// Where possible, this will provided yaml comments. +// +// Previous may optionally provide the bytes of a previous configuration file to replace settings in. +// The previous yaml file must be a valid configuration yaml, meaning all fields should be set. +// When previous is of length 0, the default configuration yaml will be used instead. +func Marshal(config *Config, previous []byte) ([]byte, error) { + if len(previous) == 0 { + previous = configBytes + } + + // load the template yaml + template := new(yaml.Node) + if err := yaml.Unmarshal(previous, template); err != nil { + return nil, err + } + + // load the config yaml + cfg, err := yamlx.Marshal(config) + if err != nil { + return nil, err + } + + // transplant the configuration yaml into the template + if err := yamlx.Transplant(template, cfg); err != nil { + return nil, err + } + + // marshal it again as a set of bytes + return yaml.Marshal(template) +} + // CSRFSecret return the csrfSecret derived from the session secret func (config *Config) CSRFSecret() []byte { // take the hash of the secret diff --git a/internal/config/config_template.yml b/internal/config/config.yml similarity index 84% rename from internal/config/config_template.yml rename to internal/config/config.yml index 42f38f9..17a08f2 100644 --- a/internal/config/config_template.yml +++ b/internal/config/config.yml @@ -1,17 +1,19 @@ paths: - # Several files are required to manage the system - # On top of this all real-system space will be created under this directory. - root: ${DEPLOY_ROOT} + # A WissKI Distillery needs to store a lot of data on disk. + # This setting defines a root folder all of these will be placed in. + # On top of this all real-system space will be created under this directory + root: "" - # You can override individual URLS in the homepage. - # Do this by adding URLs (without trailing '/'s) into a JSON file. - # This is the path to that file. - overrides: ${SELF_OVERRIDES_FILE} + # Individual paths on the root domain can be overwritten. + # This can be achieved by adding URLs (without trailing '/'s) into a JSON file. + # This setting defines the path to that file. + overrides: "" # You can block specific prefixes within Triplestore from showing up in the global resolver. # Do this by adding one prefix per line in this file. # Lines starting with '#' and blank lines are ignored. - blocks: ${SELF_RESOLVER_BLOCK_FILE} + # This setting defines the path to that file. + blocks: "" http: # Each created Drupal Instance corresponds to a single domain name. @@ -30,7 +32,8 @@ http: # By default, the default domain redirects to the distillery repository. # If you want to change this, set an alternate domain name here. -home: "" +theme: + home: "" docker: # The name of the (global) docker network to run the distillery services in. diff --git a/internal/config/database.go b/internal/config/database.go index df4a12b..6b4d9ba 100644 --- a/internal/config/database.go +++ b/internal/config/database.go @@ -8,7 +8,7 @@ type DatabaseConfig struct { // Prefix for new users and data setss UserPrefix string `yaml:"user_prefix" default:"wisski-distillery-" validate:"slug"` - DataPrefix string `yaml:"fragment_prefix" default:"wisski-distillery-" validate:"slug"` + DataPrefix string `yaml:"data_prefix" default:"wisski-distillery-" validate:"slug"` } type SQLConfig struct { diff --git a/internal/config/read.go b/internal/config/read.go index f82352f..0bf4c3b 100644 --- a/internal/config/read.go +++ b/internal/config/read.go @@ -21,7 +21,12 @@ func (config *Config) Unmarshal(env environment.Environment, src io.Reader) erro } } - // do the validator + // TODO: should this be done seperatly? + return config.Validate(env) +} + +// Validate validates this configuration file and sets appropriate defaults +func (config *Config) Validate(env environment.Environment) error { return validator.Validate(config, validators.New(env)) } diff --git a/internal/config/template.go b/internal/config/template.go index 088acc5..b30bb07 100644 --- a/internal/config/template.go +++ b/internal/config/template.go @@ -1,19 +1,13 @@ package config import ( - "bytes" - "io" "path/filepath" - "reflect" + "time" "github.com/FAU-CDI/wisski-distillery/internal/bootstrap" "github.com/FAU-CDI/wisski-distillery/pkg/environment" "github.com/FAU-CDI/wisski-distillery/pkg/hostname" "github.com/FAU-CDI/wisski-distillery/pkg/password" - "github.com/FAU-CDI/wisski-distillery/pkg/unpack" - "gopkg.in/yaml.v3" - - _ "embed" ) // Template is a template for the configuration file @@ -88,28 +82,47 @@ func (tpl *Template) SetDefaults(env environment.Environment) (err error) { return nil } -//go:embed config_template.yml -var templateBytes []byte +// Generate generates a configuration file for this configuration +func (tpl Template) Generate() Config { + return Config{ + Paths: PathsConfig{ + Root: tpl.DeployRoot, + OverridesJSON: tpl.SelfOverridesFile, + ResolverBlocks: tpl.SelfResolverBlockFile, + }, + HTTP: HTTPConfig{ + PrimaryDomain: tpl.DefaultDomain, + ExtraDomains: []string{}, + }, + Docker: DockerConfig{ + tpl.DockerNetworkName, + }, + SQL: SQLConfig{ + DatabaseConfig: DatabaseConfig{ + AdminUsername: tpl.MysqlAdminUsername, + AdminPassword: tpl.MysqlAdminPassword, -// MarshalTo marshals this template into dst -func (tpl Template) MarshalTo(dst io.Writer) error { - tplVal := reflect.ValueOf(tpl) - tplType := reflect.TypeOf(tpl) + UserPrefix: "mysql-factory-", + DataPrefix: "mysql-factory-", + }, - context := make(map[string]string, tplType.NumField()) - for i := 0; i < tplType.NumField(); i++ { - field := tplType.Field(i) + Database: "distillery", + }, + TS: TSConfig{ + DatabaseConfig: DatabaseConfig{ + AdminUsername: tpl.TriplestoreAdminUser, + AdminPassword: tpl.TriplestoreAdminPassword, - key := field.Tag.Get("env") - value := tplVal.FieldByName(field.Name).Interface() + UserPrefix: "graphdb-factory-", + DataPrefix: "graphdb-factory-", + }, + }, + MaxBackupAge: 30 * 24 * time.Hour, // 1 month + PasswordLength: 64, - bytes, err := yaml.Marshal(value) - if err != nil { - return err - } - context[key] = string(bytes) + PublicSSHPort: 2222, + + SessionSecret: tpl.SessionSecret, + CronInterval: 10 * time.Minute, } - - // TODO: CONFIG: Update template writing - return unpack.WriteTemplate(dst, context, bytes.NewReader(templateBytes)) }