diff --git a/.gitignore b/.gitignore index 13ac8d0..30c4fa0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/wdcli /distillery/overrides.json authorized_keys .vagrant diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..1e2ba9e --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +golang 1.18.5 diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..34889bb --- /dev/null +++ b/TODO.md @@ -0,0 +1,52 @@ +# WissKI-Distillery in Go + +This document describes the process of moving the distillery to using golang for the control plane (outside of docker containers). + +## Bootstrapping + +This documents the bootstraping process. +Work in progress. + +- `wdcli bootstrap $DIRECTORY` + 0. Create the deployment directory + 1. Copy over the executable (unless it already exists) + 2. Create a default configuration file (unless it already exists) + 3. Store the directory in a file called .wdcli in the $HOME directory + +- `wdcli system_update` + - to be documented +## Future Work + +- Move `provision_entrypoint.sh` into go +- Avoid running `docker compose` executable and shift it to a library +- Automatically bootstrap the docker container sql connection (use proper environment variables) +- Make error handling consistent +- Add a server that serves information +- Migrate the individual commands below +- restructure resource files +- Documentation + +## Migrating Individual Commands +- [ ] backup_all.sh +- [ ] backup_instance.sh +- [x] blind_update.sh +- [x] blind_update_all.sh +- [x] cron_all.sh +- [x] info.sh +- [x] ls.sh +- [x] make_mysql_account.sh +- [ ] monday_full.sh +- [ ] monday_short.sh +- [x] mysql.sh +- [x] provision.sh +- [x] purge.sh +- [x] rebuild.sh +- [x] rebuild_all.sh +- [x] reserve.sh +- [x] shell.sh +- [x] system_install.sh +- [x] system_update.sh +- [x] update_prefix_config.sh + +## TO BE REMOVED +- [ ] call_update_php_hack.sh diff --git a/cmd/blind_update.go b/cmd/blind_update.go new file mode 100644 index 0000000..c62dc64 --- /dev/null +++ b/cmd/blind_update.go @@ -0,0 +1,54 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/tkw1536/goprogram/exit" +) + +// BlindUpdate is the 'blind-update' command +var BlindUpdate wisski_distillery.Command = blindUpdate{} + +type blindUpdate struct { + Force bool `short:"f" long:"force" description:"force running blind-update even if AutoBlindUpdate is set to false"` + Positionals struct { + Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run blind-update in"` + } `positional-args:"true"` +} + +func (blindUpdate) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "blind_update", + Description: "Runs the blind update in the provided instances", + } +} + +var errBlindUpdateFailed = exit.Error{ + Message: "Failed to run blind update script for instance %q: exited with code %s", + ExitCode: exit.ExitGeneric, +} + +func (bu blindUpdate) Run(context wisski_distillery.Context) error { + instances, err := context.Environment.Instances(bu.Positionals.Slug...) + if err != nil { + return err + } + + for _, instance := range instances { + if !(instance.IsBlindUpdateEnabled() || bu.Force) { + context.EPrintf("skipping instance %q\n", instance.Slug) + continue + } + context.EPrintf("Updating instance %s\n", instance.Slug) + + code := instance.Shell(context.IOStream, "/utils/blind_update.sh") + if code != 0 { + return errBlindUpdateFailed.WithMessageF(instance.Slug, code) + } + } + + return nil +} diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go new file mode 100644 index 0000000..c6515c9 --- /dev/null +++ b/cmd/bootstrap.go @@ -0,0 +1,187 @@ +package cmd + +import ( + "io/fs" + "os" + "path/filepath" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/distillery" + "github.com/FAU-CDI/wisski-distillery/env" + cfg "github.com/FAU-CDI/wisski-distillery/internal/config" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/hostname" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/password" + "github.com/tkw1536/goprogram/exit" +) + +// Bootstrap is the 'bootstrap' command +var Bootstrap wisski_distillery.Command = bootstrap{} + +type bootstrap struct { + Directory string `short:"r" long:"root-directory" description:"path to the root deployment directory" default:"/var/www/deploy"` + Hostname string `short:"h" long:"hostname" description:"default hostname of the distillery (default: system hostname)"` +} + +func (bootstrap) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: false, + }, + Command: "bootstrap", + Description: "Bootstraps the installation of a Distillery System", + } +} + +var errBootstrapDifferent = exit.Error{ + Message: "refusing to bootstrap: base directory is already set to %s.", + ExitCode: exit.ExitGeneric, +} + +var errBootstrapFailedToCreateDirectory = exit.Error{ + Message: "failed to create directory %s", + ExitCode: exit.ExitGeneric, +} + +var errBootstrapFailedToSaveDirectory = exit.Error{ + Message: "failed to register base directory: %s", + ExitCode: exit.ExitGeneric, +} + +var errBoostrapFailedToCopyExe = exit.Error{ + Message: "failed to copy wdcli executable: %s", + ExitCode: exit.ExitGeneric, +} + +var errBootstrapWriteConfig = exit.Error{ + Message: "failed to write configuration file: %s", + ExitCode: exit.ExitGeneric, +} + +var errBootstrapOpenConfig = exit.Error{ + Message: "failed to open configuration file: %s", + ExitCode: exit.ExitGeneric, +} + +var errBootstrapCreateFile = exit.Error{ + Message: "failed to touch configuration file: %s", + ExitCode: exit.ExitGeneric, +} + +func (bs bootstrap) Run(context wisski_distillery.Context) error { + root := bs.Directory + + // check that we didn't get a different base directory + { + got, err := env.ReadBaseDirectory() + if err == nil && got != "" && got != root { + return errBootstrapDifferent.WithMessageF(got) + } + } + + { + logging.LogMessage(context.IOStream, "Creating root deployment directory") + if err := os.MkdirAll(root, fs.ModeDir); err != nil { + return errBootstrapFailedToCreateDirectory.WithMessageF(root) + } + if err := env.WriteBaseDirectory(root); err != nil { + return errBootstrapFailedToSaveDirectory.WithMessageF(root) + } + context.Println(root) + } + + // TODO: Read these from the command line? + wdcliPath := filepath.Join(root, "wdcli") + envPath := filepath.Join(root, ".env") + domain := bs.Hostname + if domain == "" { + domain = hostname.FQDN() + } + overridesPath := filepath.Join(root, "overrides.json") + authorizedKeysFile := filepath.Join(root, "authorized_keys") + + { + logging.LogMessage(context.IOStream, "Copying over wdcli executable") + exe, err := os.Executable() + if err != nil { + return errBoostrapFailedToCopyExe.WithMessageF(err) + } + + err = fsx.CopyFile(wdcliPath, exe) + if err != nil && err != fsx.ErrCopySameFile { + return errBoostrapFailedToCopyExe.WithMessageF(err) + } + context.Println(wdcliPath) + } + + { + if !fsx.IsFile(envPath) { + if err := logging.LogOperation(func() error { + password, err := password.Password(128) + if err != nil { + return errBootstrapWriteConfig.WithMessageF(err) + } + + if err := distillery.InstallTemplate(envPath, filepath.Join("resources", "templates", "bootstrap", "env"), map[string]string{ + "DEPLOY_ROOT": root, + "DEFAULT_DOMAIN": domain, + "SELF_OVERRIDES_FILE": overridesPath, + "AUTHORIZED_KEYS_FILE": authorizedKeysFile, + + "GRAPHDB_ADMIN_USER": "admin", + "GRAPHDB_ADMIN_PASSWORD": password[:64], + + "MYSQL_ADMIN_USER": "admin", + "MYSQL_ADMIN_PASSWORD": password[64:], + }); err != nil { + return errBootstrapWriteConfig.WithMessageF(err) + } + + return nil + }, context.IOStream, "Installing configuration file"); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + + context.Println(overridesPath) + if err := distillery.InstallTemplate(overridesPath, filepath.Join("resources", "templates", "bootstrap", "overrides.json"), map[string]string{}); err != nil { + return errBootstrapCreateFile.WithMessageF(err) + } + + context.Println(authorizedKeysFile) + if err := distillery.InstallTemplate(authorizedKeysFile, filepath.Join("resources", "templates", "bootstrap", "global_authorized_keys"), map[string]string{}); err != nil { + return errBootstrapCreateFile.WithMessageF(err) + } + + return nil + }, context.IOStream, "Creating additional config files"); err != nil { + return err + } + } + + } + + // re-read the configuration and print it! + logging.LogMessage(context.IOStream, "Configuration is now complete") + f, err := os.Open(envPath) + if err != nil { + return errBootstrapOpenConfig.WithMessageF(err) + } + defer f.Close() + + var config cfg.Config + if err := config.Unmarshal(f); err != nil { + return errBootstrapOpenConfig.WithMessageF(err) + } + context.Println(config) + + // Tell the user how to proceed + logging.LogMessage(context.IOStream, "Bootstrap is complete") + context.Printf("Adjust the configuration file at %s\n", envPath) + context.Printf("Then grab a GraphDB zipped source file and run:\n") + context.Printf("%s system_update /path/to/graphdb.zip\n", wdcliPath) + + return nil +} diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..e1b499b --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,27 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" +) + +// Config is the configuration command +var Config wisski_distillery.Command = config{} + +type config struct { +} + +func (s config) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "config", + Description: "Prints information about configuration", + } +} + +func (s config) Run(context wisski_distillery.Context) error { + context.Printf("%#v", context.Environment.Config) + return nil +} diff --git a/cmd/cron.go b/cmd/cron.go new file mode 100644 index 0000000..33cc6d8 --- /dev/null +++ b/cmd/cron.go @@ -0,0 +1,55 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/tkw1536/goprogram/exit" +) + +// Cron is the 'cron' command +var Cron wisski_distillery.Command = cron{} + +type cron struct { + Positionals struct { + Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run cron in"` + } `positional-args:"true"` +} + +func (cron) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "cron", + Description: "Runs the cron script for several instances", + } +} + +var errCronFailed = exit.Error{ + Message: "Failed to run cron script for instance %q: exited with code %s", + ExitCode: exit.ExitGeneric, +} + +func (cr cron) Run(context wisski_distillery.Context) error { + instances, err := context.Environment.Instances(cr.Positionals.Slug...) + if err != nil { + return err + } + + // iterate over the instances and store the last value of error + for _, instance := range instances { + logging.LogOperation(func() error { + code := instance.Shell(context.IOStream, "/utils/cron.sh") + if code != 0 { + // keep going, because we want to run as many crons as possible + err = errBlindUpdateFailed.WithMessageF(instance.Slug, code) + context.EPrintln(err) + } + + return nil + }, context.IOStream, "running cron for instance %s", instance.Slug) + } + + return err +} diff --git a/cmd/info.go b/cmd/info.go new file mode 100644 index 0000000..89de7dd --- /dev/null +++ b/cmd/info.go @@ -0,0 +1,45 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" +) + +// Info is then 'info' command +var Info wisski_distillery.Command = info{} + +type info struct { + Positionals struct { + Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show info about"` + } `positional-args:"true"` +} + +func (info) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "info", + Description: "Provide information about a single repository", + } +} + +func (i info) Run(context wisski_distillery.Context) error { + instance, err := context.Environment.Instance(i.Positionals.Slug) + if err != nil { + return err + } + + context.Printf("URL: %s\n", instance.URL()) + context.Printf("Base directory: %s\n", instance.FilesystemBase) + + context.Printf("SQL Database: %s\n", instance.SqlDatabase) + context.Printf("SQL Username: %s\n", instance.SqlUser) + context.Printf("SQL Password: %s\n", instance.SqlPassword) + + context.Printf("GraphDB Repository: %s\n", instance.GraphDBRepository) + context.Printf("GraphDB Username: %s\n", instance.GraphDBUser) + context.Printf("GraphDB Password: %s\n", instance.GraphDBPassword) + + return nil +} diff --git a/cmd/license.go b/cmd/license.go new file mode 100644 index 0000000..7195680 --- /dev/null +++ b/cmd/license.go @@ -0,0 +1,47 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/legal" +) + +// License is the 'wdcli license' command. +// +// The license command prints to standard output legal notices about the wdcli program. +var License wisski_distillery.Command = license{} + +type license struct{} + +func (license) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: false, + }, + Command: "license", + Description: "Print licensing information about wdcli and exit. ", + } +} + +func (license) AfterParse() error { + return nil +} + +func (license) Run(context wisski_distillery.Context) error { + context.Printf(stringLicenseInfo, wisski_distillery.License, legal.Notices) + return nil +} + +const stringLicenseInfo = ` +wdcli -- WissKI Distillery Command Line Utility +https://github.com/FAU-CDI/wisski-distillery + +================================================================================ +wdcli is licensed under the terms of the AGPL Version 3.0 License: + +%s +================================================================================ + +Furthermore, this executable may include code from the following projects: +%s +` diff --git a/cmd/ls.go b/cmd/ls.go new file mode 100644 index 0000000..ddc56cd --- /dev/null +++ b/cmd/ls.go @@ -0,0 +1,38 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" +) + +// Ls is the 'ls' command +var Ls wisski_distillery.Command = ls{} + +type ls struct { + Positionals struct { + Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug(s) of instance(s) to list"` + } `positional-args:"true"` +} + +func (ls) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "ls", + Description: "Lists WissKI instances", + } +} + +func (l ls) Run(context wisski_distillery.Context) error { + instances, err := context.Environment.Instances(l.Positionals.Slug...) + if err != nil { + return err + } + + for _, instance := range instances { + context.Println(instance.Slug) + } + + return nil +} diff --git a/cmd/make_mysql_account.go b/cmd/make_mysql_account.go new file mode 100644 index 0000000..bacf6c5 --- /dev/null +++ b/cmd/make_mysql_account.go @@ -0,0 +1,67 @@ +package cmd + +import ( + "fmt" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/sqle" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/parser" +) + +// Shell is the 'shell' command +var MakeMysqlAccount wisski_distillery.Command = makeMysqlAccount{} + +type makeMysqlAccount struct{} + +func (makeMysqlAccount) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + ParserConfig: parser.Config{ + IncludeUnknown: true, + }, + Command: "make_mysql_account", + Description: "Open a shell in the provided instance", + } +} + +var errUnableToReadUsername = exit.Error{ + ExitCode: exit.ExitGeneric, + Message: "unable to read username: %s", +} + +var errUnableToReadPassword = exit.Error{ + ExitCode: exit.ExitGeneric, + Message: "unable to read password: %s", +} + +func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error { + context.Printf("Username>") + username, err := context.ReadLine() + if err != nil { + return errUnableToReadUsername.WithMessageF(err) + } + + context.Printf("Password>") + password, err := context.ReadPassword() + if err != nil { + return errUnableToReadPassword.WithMessageF(err) + } + + query := sqle.Format("CREATE USER ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) + if err != nil { + return err + } + code := context.Environment.SQLShell(context.IOStream, "-e", query) + + if code != 0 { + return exit.Error{ + ExitCode: exit.ExitCode(uint8(code)), + Message: fmt.Sprintf("Exit code %d", code), + } + } + return nil +} diff --git a/cmd/mysql.go b/cmd/mysql.go new file mode 100644 index 0000000..1b9ed29 --- /dev/null +++ b/cmd/mysql.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "fmt" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/parser" +) + +// Mysql is the 'mysql' command +var Mysql wisski_distillery.Command = mysql{} + +type mysql struct { + Positionals struct { + Args []string `positional-arg-name:"ARGS" description:"arguments to pass to the mysql command"` + } `positional-args:"true"` +} + +func (mysql) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + ParserConfig: parser.Config{ + IncludeUnknown: true, + }, + Command: "mysql", + Description: "Opens a mysql shell", + } +} + +func (ms mysql) Run(context wisski_distillery.Context) error { + code := context.Environment.SQLShell(context.IOStream, ms.Positionals.Args...) + if code != 0 { + return exit.Error{ + ExitCode: exit.ExitCode(uint8(code)), + Message: fmt.Sprintf("Exit code %d", code), + } + } + return nil +} diff --git a/cmd/prefix.go b/cmd/prefix.go new file mode 100644 index 0000000..f2bd88b --- /dev/null +++ b/cmd/prefix.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "io/fs" + "os" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/tkw1536/goprogram/exit" +) + +// Cron is the 'cron' command +var UpdatePrefixConfig wisski_distillery.Command = updateprefixconfig{} + +type updateprefixconfig struct{} + +func (updateprefixconfig) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "update_prefix_config", + Description: "Updates the prefix configuration", + } +} + +var errPrefixUpdateFailed = exit.Error{ + Message: "Failed to update the prefix configuration: %s", + ExitCode: exit.ExitGeneric, +} + +func (upc updateprefixconfig) Run(context wisski_distillery.Context) error { + dis := context.Environment + + instances, err := dis.AllInstances() + if err != nil { + return errPrefixUpdateFailed.WithMessageF(err) + } + + target := dis.ResolverPrefixConfig() + + // print the configuration + config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm) + if err != nil { + return errPrefixUpdateFailed.WithMessageF(err) + } + + // iterate over the instances and store the last value of error + for _, instance := range instances { + if err := logging.LogOperation(func() error { + // read the prefix config + data, err := instance.PrefixConfig() + if err != nil { + return err + } + context.IOStream.Printf("%s", data) + + // and write it out! + if _, err := config.WriteString(data); err != nil { + return err + } + + return nil + }, context.IOStream, "reading prefix config %s", instance.Slug); err != nil { + return errPrefixUpdateFailed.WithMessageF(err) + } + } + + // and restart the resolver to apply the config! + logging.LogMessage(context.IOStream, "restarting resolver stack") + if err := dis.ResolverStack().Restart(context.IOStream); err != nil { + return errPrefixUpdateFailed.WithMessageF(err) + } + + return err +} diff --git a/cmd/provision.go b/cmd/provision.go new file mode 100644 index 0000000..022288c --- /dev/null +++ b/cmd/provision.go @@ -0,0 +1,121 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/tkw1536/goprogram/exit" +) + +// Provision is the 'provision' command +var Provision wisski_distillery.Command = provision{} + +type provision struct { + Positionals struct { + Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to create"` + } `positional-args:"true"` +} + +func (provision) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "provision", + Description: "Creates a new WissKI Instance", + } +} + +// TODO: AfterParse to check instance! + +var errProvisionAlreadyExists = exit.Error{ + Message: "Instance %q already exists", + ExitCode: exit.ExitGeneric, +} + +var errProvisionGeneric = exit.Error{ + Message: "Unable to provision instance %s: %s", + ExitCode: exit.ExitGeneric, +} + +func (p provision) Run(context wisski_distillery.Context) error { + dis := context.Environment + slug := p.Positionals.Slug + + // check that it doesn't already exist + logging.LogMessage(context.IOStream, "Provisioning new WissKI instance %s", slug) + if exists, err := dis.HasInstance(slug); err != nil || exists { + return errProvisionAlreadyExists.WithMessageF(slug) + } + + // make it in-memory + instance, err := dis.NewInstance(slug) + if err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + // check that the base directory does not exist + logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase) + if fsx.IsDirectory(instance.FilesystemBase) { + return errProvisionAlreadyExists.WithMessageF(slug) + } + + // Store in bookkeeping + if err := logging.LogOperation(func() error { + if err := instance.Update(); err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + return nil + }, context.IOStream, "Updating bookkeeping database"); err != nil { + return err + } + + // create the sql + if err := logging.LogOperation(func() error { + if err := dis.SQLProvision(instance.SqlDatabase, instance.SqlUser, instance.SqlPassword); err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + return nil + }, context.IOStream, "Provisioning SQL Database"); err != nil { + return err + } + + // create the triplestore + if err := logging.LogOperation(func() error { + if err := dis.TriplestoreProvision(instance.GraphDBRepository, instance.Domain(), instance.GraphDBUser, instance.GraphDBPassword); err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + return nil + }, context.IOStream, "Provisioning Triplestore"); err != nil { + return err + } + + // run the provision script + if err := logging.LogOperation(func() error { + if err := instance.Provision(context.IOStream); err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + return nil + }, context.IOStream, "Running setup scripts"); err != nil { + return err + } + + // start the container! + logging.LogMessage(context.IOStream, "Starting Container") + if err := instance.Stack().Up(context.IOStream); err != nil { + return err + } + + // and we're done! + logging.LogMessage(context.IOStream, "Instance has been provisioned") + context.Printf("URL: %s\n", instance.URL().String()) + context.Printf("Username: %s\n", instance.DrupalUsername) + context.Printf("Password: %s\n", instance.DrupalPassword) + + return nil +} diff --git a/cmd/purge.go b/cmd/purge.go new file mode 100644 index 0000000..e7baf57 --- /dev/null +++ b/cmd/purge.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "os" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/tkw1536/goprogram/exit" +) + +// Provision is the 'provision' command +var Purge wisski_distillery.Command = purge{} + +type purge struct { + Yes bool `short:"y" long:"yes" description:"Skip asking for confirmation"` + Positionals struct { + Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to purge"` + } `positional-args:"true"` +} + +func (purge) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "purge", + Description: "Purges a WissKI Instance", + } +} + +var errPurgeNoDetails = exit.Error{ + Message: "Unable to find instance details for purge: %s", + ExitCode: exit.ExitGeneric, +} + +var errPurgeNoConfirmation = exit.Error{ + Message: "Aborting after request was not confirmed. Either type `yes` or pass `--yes` on the command line", + ExitCode: exit.ExitGeneric, +} + +func (p purge) Run(context wisski_distillery.Context) error { + dis := context.Environment + slug := p.Positionals.Slug + + // check the confirmation from the user + if !p.Yes { + context.Printf("About to remove repository %s. This cannot be undone.\n", slug) + context.Printf("Type 'yes' to continue: ") + line, err := context.ReadLine() + if err != nil || line != "yes" { + return errPurgeNoConfirmation + } + } + + // load the instance (first via bookkeeping, then via defaults) + logging.LogMessage(context.IOStream, "Checking bookkeeping table") + instance, err := dis.Instance(slug) + if err == env.ErrInstanceNotFound { + context.Println("Not found in bookkeeping table, assuming defaults") + instance, err = dis.NewInstance(slug) + } + if err != nil { + return errPurgeNoDetails.WithMessageF(err) + } + + // remove docker stack + logging.LogMessage(context.IOStream, "Stopping and removing docker container") + if err := instance.Stack().Down(context.IOStream); err != nil { + context.EPrintln(err) + } + + // remove the filesystem + logging.LogMessage(context.IOStream, "Removing from filesystem %s", instance.FilesystemBase) + if err := os.RemoveAll(instance.FilesystemBase); err != nil { + context.EPrintln(err) + } + + // remove the triplestore + logging.LogOperation(func() error { + logging.LogMessage(context.IOStream, "Removing user %s", instance.GraphDBUser) + if err := dis.TriplestorePurgeUser(instance.GraphDBUser); err != nil { + context.EPrintln(err) + } + + logging.LogMessage(context.IOStream, "Removing repository %s", instance.GraphDBRepository) + if err := dis.TriplestorePurgeRepo(instance.GraphDBRepository); err != nil { + context.EPrintln(err) + } + + return nil + }, context.IOStream, "Removing from Triplestore") + + // remove the sql + logging.LogOperation(func() error { + logging.LogMessage(context.IOStream, "Removing user %s", instance.SqlUser) + if err := dis.SQLPurgeUser(instance.SqlUser); err != nil { + context.EPrintln(err) + } + + logging.LogMessage(context.IOStream, "Removing database %s", instance.SqlDatabase) + if err := dis.SQLPurgeDatabase(instance.SqlDatabase); err != nil { + context.EPrintln(err) + } + + return nil + }, context.IOStream, "Removing from SQL") + + // remove from bookkeeping + logging.LogMessage(context.IOStream, "Removing instance from bookkeeping") + if err := instance.Delete(); err != nil { + context.EPrintln(err) + } + + logging.LogMessage(context.IOStream, "Instance %s has been purged", slug) + return nil +} diff --git a/cmd/rebuild.go b/cmd/rebuild.go new file mode 100644 index 0000000..bcb39c9 --- /dev/null +++ b/cmd/rebuild.go @@ -0,0 +1,65 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/tkw1536/goprogram/exit" +) + +// Cron is the 'cron' command +var Rebuild wisski_distillery.Command = rebuild{} + +type rebuild struct { + Positionals struct { + Slug []string `positional-arg-name:"SLUG" required:"0" description:"slug of instance(s) to run rebuild"` + } `positional-args:"true"` +} + +func (rebuild) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "rebuild", + Description: "Runs the rebuild script for several instances", + } +} + +var errRebuildFailed = exit.Error{ + Message: "Failed to run rebuild script for instance %q: exited with code %s", + ExitCode: exit.ExitGeneric, +} + +func (rb rebuild) Run(context wisski_distillery.Context) error { + instances, err := context.Environment.Instances(rb.Positionals.Slug...) + if err != nil { + return err + } + + // iterate over the instances and store the last value of error + var globalErr error + for _, instance := range instances { + logging.LogOperation(func() error { + s := instance.Stack() + if err := logging.LogOperation(func() error { + return s.Install(context.IOStream, stack.InstallationContext{}) + }, context.IOStream, "Installing docker stack"); err != nil { + globalErr = err + return err + } + + if err := logging.LogOperation(func() error { + return s.Update(context.IOStream, true) + }, context.IOStream, "Updating docker stack"); err != nil { + globalErr = err + return err + } + + return nil + }, context.IOStream, "Rebuilding instance %s", instance.Slug) + } + + return globalErr +} diff --git a/cmd/reserve.go b/cmd/reserve.go new file mode 100644 index 0000000..415742b --- /dev/null +++ b/cmd/reserve.go @@ -0,0 +1,86 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/tkw1536/goprogram/exit" +) + +// Reserve is the 'reserve' command +var Reserve wisski_distillery.Command = reserve{} + +type reserve struct { + Positionals struct { + Slug string `positional-arg-name:"slug" required:"1-1" description:"name of WissKI Instance to reserve"` + } `positional-args:"true"` +} + +func (reserve) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + Command: "reserve", + Description: "Reserves a new WissKI Instance", + } +} + +// TODO: AfterParse to check instance! + +var errReserveAlreadyExists = exit.Error{ + Message: "Instance %q already exists", + ExitCode: exit.ExitGeneric, +} + +var errReserveGeneric = exit.Error{ + Message: "Unable to provision instance %s: %s", + ExitCode: exit.ExitGeneric, +} + +func (r reserve) Run(context wisski_distillery.Context) error { + dis := context.Environment + slug := r.Positionals.Slug + + // check that it doesn't already exist + logging.LogMessage(context.IOStream, "Reserving new WissKI instance %s", slug) + if exists, err := dis.HasInstance(slug); err != nil || exists { + return errProvisionAlreadyExists.WithMessageF(slug) + } + + // make it in-memory + instance, err := dis.NewInstance(slug) + if err != nil { + return errProvisionGeneric.WithMessageF(slug, err) + } + + // check that the base directory does not exist + logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase) + if fsx.IsDirectory(instance.FilesystemBase) { + return errProvisionAlreadyExists.WithMessageF(slug) + } + + // setup docker stack + s := instance.ReserveStack() + { + if err := logging.LogOperation(func() error { + return s.Install(context.IOStream, stack.InstallationContext{}) + }, context.IOStream, "Installing docker stack"); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + return s.Update(context.IOStream, true) + }, context.IOStream, "Updating docker stack"); err != nil { + return err + } + } + + // and we're done! + logging.LogMessage(context.IOStream, "Instance has been reserved") + context.Printf("URL: %s\n", instance.URL().String()) + + return nil +} diff --git a/cmd/shell.go b/cmd/shell.go new file mode 100644 index 0000000..ebd67fb --- /dev/null +++ b/cmd/shell.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "fmt" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/parser" +) + +// Shell is the 'shell' command +var Shell wisski_distillery.Command = shell{} + +type shell struct { + Positionals struct { + Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show run shell in"` + Args []string `positional-arg-name:"ARGS" description:"arguments to pass to the shell"` + } `positional-args:"true"` +} + +func (shell) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + ParserConfig: parser.Config{ + IncludeUnknown: true, + }, + Command: "shell", + Description: "Open a shell in the provided instance", + } +} + +func (sh shell) Run(context wisski_distillery.Context) error { + instance, err := context.Environment.Instance(sh.Positionals.Slug) + if err != nil { + return err + } + + code := instance.Shell(context.IOStream, sh.Positionals.Args...) + if code != 0 { + return exit.Error{ + ExitCode: exit.ExitCode(uint8(code)), + Message: fmt.Sprintf("Exit code %d", code), + } + } + return nil +} diff --git a/cmd/system_update.go b/cmd/system_update.go new file mode 100644 index 0000000..8b65bbe --- /dev/null +++ b/cmd/system_update.go @@ -0,0 +1,182 @@ +package cmd + +import ( + "os" + "path/filepath" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/distillery" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/FAU-CDI/wisski-distillery/internal/execx" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/parser" +) + +// SystemUpdate is the 'system_update' command +var SystemUpdate wisski_distillery.Command = systemupdate{} + +type systemupdate struct { + SkipCoreUpdates bool `short:"s" long:"skip-core-updates" description:"Skip applying operating system and other core system updates"` + Positionals struct { + GraphdbZip string `positional-arg-name:"PATH_TO_GRAPHDB_ZIP" required:"1-1" description:"path to the graphdb.zip file"` + } `positional-args:"true"` +} + +func (systemupdate) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: env.Requirements{ + NeedsConfig: true, + }, + ParserConfig: parser.Config{ + IncludeUnknown: true, + }, + Command: "system_update", + Description: "Installs and Update Components of the WissKI Distillery System", + } +} + +var errNoGraphDBZip = exit.Error{ + Message: "%s does not exist", + ExitCode: exit.ExitCommandArguments, +} + +func (s systemupdate) AfterParse() error { + _, err := os.Stat(s.Positionals.GraphdbZip) + if os.IsNotExist(err) { + return errNoGraphDBZip.WithMessageF(s.Positionals.GraphdbZip) + } + if err != nil { + return err + } + return nil +} + +var errFailedToCreateDirectory = exit.Error{ + Message: "failed to create directory %s: %s", + ExitCode: exit.ExitGeneric, +} + +var errFailedRuntime = exit.Error{ + Message: "failed to update runtime: %s", + ExitCode: exit.ExitGeneric, +} + +func (si systemupdate) Run(context wisski_distillery.Context) error { + dis := context.Environment + + // create all the other directories + logging.LogMessage(context.IOStream, "Ensuring distillery installation directories exist") + for _, d := range []string{ + dis.Config.DeployRoot, + dis.InstancesDir(), + dis.InprogressBackupPath(), + dis.FinalBackupPath(), + } { + context.Println(d) + if err := os.MkdirAll(d, os.ModeDir); err != nil { + return errFailedToCreateDirectory.WithMessageF(d, err) + } + } + + if !si.SkipCoreUpdates { + // install system updates + logging.LogMessage(context.IOStream, "Updating Operating System Packages") + if err := si.mustExec(context, "", "apt-get", "update"); err != nil { + return err + } + if err := si.mustExec(context, "", "apt-get", "upgrade", "-y"); err != nil { + return err + } + + // install docker + logging.LogMessage(context.IOStream, "Installing / Updating Docker") + if err := si.mustExec(context, "", "apt-get", "install", "curl"); err != nil { + return err + } + if err := si.mustExec(context, "", "/bin/sh", "-c", "curl -fsSL https://get.docker.com -o - | /bin/sh"); err != nil { + return err + } + } + + // create the docker network + // TODO: Use docker API for this + logging.LogMessage(context.IOStream, "Updating Docker Configuration") + si.mustExec(context, "", "docker", "network", "create", "distillery") + + // install and update the various stacks! + ctx := stack.InstallationContext{ + "graphdb.zip": si.Positionals.GraphdbZip, + } + + if err := logging.LogOperation(func() error { + for _, stack := range dis.Stacks() { + if err := logging.LogOperation(func() error { + return stack.Install(context.IOStream, ctx) + }, context.IOStream, "Installing docker stack %q", stack.Name); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + return stack.Update(context.IOStream, true) + }, context.IOStream, "Updating docker stack %q", stack.Name); err != nil { + return err + } + } + return nil + }, context.IOStream, "Updating Components"); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + if err := distillery.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), func(dst, src string) { + context.Printf("[copy] %s\n", dst) + }); err != nil { + return errFailedRuntime.WithMessageF(err) + } + return nil + }, context.IOStream, "Unpacking Runtime Components"); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + if err := dis.SQLBootstrap(context.IOStream); err != nil { + return err + } + return nil + }, context.IOStream, "Bootstraping SQL database"); err != nil { + return err + } + + if err := logging.LogOperation(func() error { + if err := dis.TriplestoreBootstrap(context.IOStream); err != nil { + return err + } + return nil + }, context.IOStream, "Bootstraping Triplestore"); err != nil { + return err + } + + logging.LogMessage(context.IOStream, "System has been updated") + return nil +} + +var errMustExecFailed = exit.Error{ + Message: "process exited with code %d", +} + +// mustExec indicates that the given executable process must complete successfully. +// If it does not, returns errMustExecFailed +func (si systemupdate) mustExec(context wisski_distillery.Context, workdir string, exe string, argv ...string) error { + if workdir == "" { + workdir = context.Environment.Config.DeployRoot + } + code := execx.Exec(context.IOStream, workdir, exe, argv...) + if code != 0 { + err := errMustExecFailed.WithMessageF(code) + err.ExitCode = exit.ExitCode(code) + return err + } + return nil +} diff --git a/cmd/wdcli/main.go b/cmd/wdcli/main.go new file mode 100644 index 0000000..609e4ec --- /dev/null +++ b/cmd/wdcli/main.go @@ -0,0 +1,110 @@ +// Command wdcli implement the entry point for the wisski-distillery +package main + +import ( + "fmt" + "os" + "runtime/debug" + + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/cmd" + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/stream" +) + +var wdcli = wisski_distillery.NewProgram() + +func init() { + // self commands + wdcli.Register(cmd.Config) + wdcli.Register(cmd.License) + + // setup commands + wdcli.Register(cmd.Bootstrap) + wdcli.Register(cmd.SystemUpdate) + + // sql commands + wdcli.Register(cmd.Mysql) + wdcli.Register(cmd.MakeMysqlAccount) + + // instance setup and teardown + wdcli.Register(cmd.Provision) + wdcli.Register(cmd.Purge) + wdcli.Register(cmd.Reserve) + wdcli.Register(cmd.Rebuild) + + // instance management + wdcli.Register(cmd.Ls) + wdcli.Register(cmd.Info) + + // instance tasks + wdcli.Register(cmd.Shell) + wdcli.Register(cmd.BlindUpdate) + wdcli.Register(cmd.Cron) + wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration + + // backup & cron + // wdcli.Register(cmd.BackupInstance) + // wdcli.Register(cmd.BackupAll) +} + +// an error when no arguments are provided. +var errNoArgumentsProvided = exit.Error{ + ExitCode: exit.ExitGeneralArguments, + Message: "Need at least one argument. Use `wdcli license` to view licensing information. ", +} + +func main() { + // recover from calls to panic(), and exit the program appropriatly. + // This has to be in the main() function because any of the library functions might be broken. + // For this reason, as few ggman functions as possible are used here; just stuff from the top-level ggman package. + defer func() { + if err := recover(); err != nil { + fmt.Fprintf(os.Stderr, fatalPanicMessage, err) + debug.PrintStack() + exit.ExitPanic.Return() + } + }() + + streams := stream.FromEnv() + + // when there are no arguments then parsing argument *will* fail + // + // we don't need to even bother with the rest of the program + // just immediatly return a custom error message. + if len(os.Args) == 1 { + streams.Die(errNoArgumentsProvided) + errNoArgumentsProvided.Return() + return + } + + // creat a new set of parameters + // and then use them to execute the main command + err := func() error { + params, err := env.ParamsFromEnv() + if err != nil { + return streams.Die(err) + } + + return wdcli.Main(streams, params, os.Args[1:]) + }() + + // return the error to the user + + exit.AsError(err).Return() +} + +const fatalPanicMessage = `Fatal Error: Panic + +The wdcli program panicked and had to abort execution. This is usually +indicative of a bug. If this occurs repeatedly you might want to consider +filing an issue in the issue tracker at: + +https://github.com/FAU-CDI/wisski-distillery/issues + +Below is debug information that might help the developers track down what +happened. + +panic: %v +` diff --git a/distillery/blind_update.sh b/distillery/blind_update.sh deleted file mode 100644 index 81cabc8..0000000 --- a/distillery/blind_update.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -require_slug_argument - - -# if the site doesn't exist, I can't open a shell. -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I can't rebuild it. " - exit 1 -fi; - -# Read everything from the database -read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)" - -# cd into the right directory -cd "$INSTANCE_BASE_DIR" - -# and open a www-data shell -docker-compose exec barrel /user_shell.sh /utils/blind_update.sh \ No newline at end of file diff --git a/distillery/blind_update_all.sh b/distillery/blind_update_all.sh deleted file mode 100644 index 84963a6..0000000 --- a/distillery/blind_update_all.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -DISABLE_LOG=0 - -# update all the instances -for slug in $(sql_bookkeep_list_updateable); do - log_info "=> /bin/bash $DIR/blind_update.sh '$slug'" - /bin/bash "$DIR/blind_update.sh" "$slug"; -done - diff --git a/distillery/cron_all.sh b/distillery/cron_all.sh deleted file mode 100644 index 0454a2a..0000000 --- a/distillery/cron_all.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -unset DISABLE_LOG - -# update all the instances -for slug in $(sql_bookkeep_list); do - read -r INSTANCE_BASE_DIR <<< "$(sql_bookkeep_load "${slug}" "filesystem_base" | tail -n +2)" - log_info "=> Runnning cron for '$slug'" - cd "$INSTANCE_BASE_DIR" - docker-compose exec barrel /bin/bash /utils/cron.sh -done - diff --git a/distillery/info.sh b/distillery/info.sh deleted file mode 100755 index 24babd0..0000000 --- a/distillery/info.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -DISABLE_LOG=0 -require_slug_argument - - -# if the site doesn't exist, I can't open a shell. -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I can't show info about it. " - exit 1 -fi; - -# Read everything from the database -read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER GRAPHDB_PASSWORD <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user,graphdb_password" | tail -n +2)" - -GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)" - -# read sql configuration -cd "$INSTANCE_BASE_DIR" -read -r SQL_DATABASE SQL_USER SQL_PASS SQL_OTHER <<< "$(docker-compose exec barrel drush sql:conf --format=tsv --show-passwords)" - -echo "==================================================================================" -echo "URL: http://$INSTANCE_DOMAIN" -echo "Base directory: ${INSTANCE_BASE_DIR}" -log_info " => Your GraphDB details (for WissKI Salz) are: " -echo "Read URL: http://triplestore:7200/repositories/$GRAPHDB_REPO" -echo "Write URL: http://triplestore:7200/repositories/$GRAPHDB_REPO/statements" -echo "Username: $GRAPHDB_USER" -echo "Password: $GRAPHDB_PASSWORD" -echo "Authorization Header: $GRAPHDB_HEADER" -echo "Writable: yes" -echo "Default Graph URI: http://$INSTANCE_DOMAIN/#" -echo "Ontology Paths: (empty)" -echo "SameAs property: http://www.w3.org/2002/07/owl#sameAs" -log_info " => Your SQL detsils are: " -echo "SQL Database: $SQL_DATABASE" -echo "SQL Username: $SQL_USER" -echo "SQL Password: $SQL_PASS" diff --git a/distillery/ls.sh b/distillery/ls.sh deleted file mode 100644 index e8c93a3..0000000 --- a/distillery/ls.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -DISABLE_LOG=0 - -# list all the existing instances -sql_bookkeep_list \ No newline at end of file diff --git a/distillery/make_mysql_account.sh b/distillery/make_mysql_account.sh deleted file mode 100644 index c26946c..0000000 --- a/distillery/make_mysql_account.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh -DISABLE_LOG=0 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" - -# wait for sql to come up -wait_for_sql > /dev/null - -echo "Creating new MySQL user with root privileges. " -read -p 'Enter Username:' MYSQL_USER -read -sp 'Enter password:' MYSQL_PASSWORD - -if ! is_valid_slug "$MYSQL_USER"; then - echo "Not a valid username: ${MYSQL_USER}" - echo "User must be alphanumeric for sql injection reasons. " - echo "You can always create a user manually. " - exit 1 -fi - -if ! is_valid_slug "$MYSQL_PASSWORD"; then - echo "Not a valid password: ${MYSQL_PASSWORD}" - echo "Password must be alphanumeric for sql injection reasons. " - echo "You can always create a user manually. " - exit 1 -fi - -dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO \`${MYSQL_USER}\`@\`%\` WITH GRANT OPTION; FLUSH PRIVILEGES;" - -log_info "Created user ${MYSQL_USER}" \ No newline at end of file diff --git a/distillery/mysql.sh b/distillery/mysql.sh deleted file mode 100755 index 421abb1..0000000 --- a/distillery/mysql.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" - -# wait for sql to come up -wait_for_sql > /dev/null -dockerized_mysql_interactive "$@" diff --git a/distillery/provision.sh b/distillery/provision.sh deleted file mode 100755 index 655b767..0000000 --- a/distillery/provision.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -require_slug_argument - -# wait for sql to be awake -wait_for_sql - -# check if the site exists -if sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' already exists in bookeeping table. " - echo "Refusing to work" - exit 1; -fi - -# Randomly generate the database name and user we will configure. -# Use the 'randompw' alias for this. -log_info " => Generating new MySQL password" -MYSQL_PASSWORD="$(randompw)" - -# Initialize the SQL database with those credentials. -log_info " => Intializing new SQL database '${MYSQL_DATABASE}' and user '$MYSQL_USER'. " -dockerized_mysql -e "CREATE DATABASE \`${MYSQL_DATABASE}\`;" -dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}';" -dockerized_mysql -e "GRANT ALL PRIVILEGES ON \`${MYSQL_DATABASE}\`.* TO \`${MYSQL_USER}\`@\`%\`;" -dockerized_mysql -e "FLUSH PRIVILEGES;" - -# Create a new repository for GraphDB. -# Use the template for this. -log_info " => Generating new GraphDB repository '$GRAPHDB_REPO'" -load_template "repository/graphdb-repo.ttl" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "INSTANCE_DOMAIN" "${INSTANCE_DOMAIN}" | \ -curl -X POST $GRAPHDB_AUTH_FLAGS \ - http://127.0.0.1:7200/rest/repositories \ - --header 'Content-Type: multipart/form-data' \ - -F "config=@-" - -# Generate a random password for the GraphDB user -log_info " => Generating a new GraphDB password" -GRAPHDB_PASSWORD="$(randompw)" - -# Create the user and grant them access to the creatd database. -log_info " => Creating GraphDB user '$GRAPHDB_USER'" -load_template "repository/graphdb-user.json" "GRAPHDB_USER" "${GRAPHDB_USER}" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "GRAPHDB_PASSWORD" "${GRAPHDB_PASSWORD}" | \ -curl -X POST $GRAPHDB_AUTH_FLAGS \ - "http://127.0.0.1:7200/rest/security/users/${GRAPHDB_USER}" \ - --header 'Content-Type: application/json' \ - --header 'Accept: text/plain' \ - -d @- - -log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'" -mkdir -p "$INSTANCE_BASE_DIR" -mkdir -p "$INSTANCE_DATA_DIR" -mkdir -p "$INSTANCE_DATA_DIR/.composer" -mkdir -p "$INSTANCE_DATA_DIR/data" -touch "$INSTANCE_DATA_DIR/authorized_keys" -chmod a+rw "$INSTANCE_DATA_DIR/authorized_keys" - -# Generate some more random credentials, this time for drupal. -# We again make use of the randompw alias. -log_info " => Generating new drupal credentials" -DRUPAL_USER="admin" -DRUPAL_PASS="$(randompw)" - -# TODO: copy over docker-compose into the right directory -log_info " => Creating instance directory" -install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR" - -# Log all the details into the bookeeping database -log_info "=> Storing configuration in bookkeeping table" -sql_bookkeep_insert \ - "slug,filesystem_base,sql_database,sql_user,sql_password,graphdb_repository,graphdb_user,graphdb_password" \ - "\"${SLUG}\",\"${INSTANCE_BASE_DIR}\",\"${MYSQL_DATABASE}\",\"${MYSQL_USER}\",\"${MYSQL_PASSWORD}\",\"${GRAPHDB_REPO}\",\"${GRAPHDB_USER}\",\"${GRAPHDB_PASSWORD}\"" - -log_info " => Writing configuration file" -load_template "docker-env/barrel" \ - "REAL_PATH" "${INSTANCE_DATA_DIR}" \ - "GLOBAL_AUTHORIZED_KEYS_FILE" "${GLOBAL_AUTHORIZED_KEYS_FILE}" \ - "VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \ - "SLUG" "${SLUG}" \ - "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \ - "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \ - "DISTILLERY_DIR" "${DIR}" \ - > "$INSTANCE_BASE_DIR/.env" - - -log_info " => Running and building image" -cd "$INSTANCE_BASE_DIR" -docker-compose build --pull -docker-compose pull - -log_info " => Running provision script" -docker-compose run --rm barrel /bin/bash -c "sudo PATH=\$PATH -u www-data /bin/bash /provision_container.sh \ - \"${INSTANCE_DOMAIN}\" \ - \"${MYSQL_DATABASE}\" \"${MYSQL_USER}\" \"${MYSQL_PASSWORD}\" \ - \"${GRAPHDB_REPO}\" \"${GRAPHDB_USER}\" \"${GRAPHDB_PASSWORD}\" \ - \"${DRUPAL_USER}\" \"${DRUPAL_PASS}\" \ - \"${DRUPAL_VERSION}\" \"${WISSKI_VERSION}\"" - - -log_info " => Starting container" -docker-compose up -d - diff --git a/distillery/purge.sh b/distillery/purge.sh deleted file mode 100755 index c19d0b0..0000000 --- a/distillery/purge.sh +++ /dev/null @@ -1,68 +0,0 @@ -# To install a new system: - -# This script will provision a new Drupal instance and make it available to apache. -# Usage: sudo ./provision.sh $SLUG -# In case the installation fails, it will bail out and leave you with an incomplete installation. -# To delete an incomplete installation, use the ./purge.sh script, or try fixing the error manually. -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -require_slug_argument - -# wait for sql to be awake -wait_for_sql - -while true; do - log_info " => I'm about to delete the '$SLUG' site from this system. " - read -p "This can not be undone. Please type 'y' to continue: " yn - case $yn in - [Yy]* ) break;; - * ) echo "Abort. "; exit 1;; - esac -done - -# check if the site exists -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I'll try to cleanup with the current defaults. " - echo "This may or may not work. " -else - # Read all the configuration from the database - log_info " => Reading components from database" - read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)" -fi - -# stop the running system container -if [ -d "$INSTANCE_BASE_DIR" ] ; then - log_info "=> Stopping running system" - cd "$INSTANCE_BASE_DIR" - docker-compose down -v || true -fi; - -cd - -# delete the mysql database. -log_info " => Deleting MySQL database '$MYSQL_DATABASE' and user '$MYSQL_USER'. " -dockerized_mysql -e "DROP DATABASE IF EXISTS \`${MYSQL_DATABASE}\`;" -dockerized_mysql -e "DROP USER IF EXISTS \`${MYSQL_USER}\`@\`%\`;" -dockerized_mysql -e "FLUSH PRIVILEGES;" - -# Clear the GraphDB repository. -log_info " => Deleting GraphDB repository '$GRAPHDB_REPO'" -curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/repositories/$GRAPHDB_REPO - -log_info " => Deleting GraphDB user '$GRAPHDB_USER'" -curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/security/users/$GRAPHDB_USER - - -# Delete the directory -log_info " => Deleting '$INSTANCE_BASE_DIR'" -rm -rf "$INSTANCE_BASE_DIR" - -log_info " => Clearing bookkeeping record" -sql_bookeep_delete "$SLUG" || true - -log_info " => '$SLUG' has been purged. " diff --git a/distillery/rebuild.sh b/distillery/rebuild.sh deleted file mode 100644 index d4711b0..0000000 --- a/distillery/rebuild.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -require_slug_argument - - -# if the site doesn't exist, I can't open a shell. -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I can't rebuild it. " - exit 1 -fi; - -# Read everything from the database -read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)" - -log_info " => Touching authorized_keys file" -touch "$INSTANCE_BASE_DIR/data/authorized_keys" -chmod a+rw "$INSTANCE_BASE_DIR/data/authorized_keys" - -log_info " => Updating compose files" -install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR" - -log_info "=> Rebuilding and restarting '$INSTANCE_BASE_DIR'" -update_stack "$INSTANCE_BASE_DIR" \ No newline at end of file diff --git a/distillery/rebuild_all.sh b/distillery/rebuild_all.sh deleted file mode 100755 index 3ba3d35..0000000 --- a/distillery/rebuild_all.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -DISABLE_LOG=0 - -# update all the instances -for slug in $(sql_bookkeep_list); do - log_info "=> /bin/bash $DIR/rebuild.sh '$slug'" - /bin/bash "$DIR/rebuild.sh" "$slug"; -done - diff --git a/distillery/reserve.sh b/distillery/reserve.sh deleted file mode 100755 index 9afe634..0000000 --- a/distillery/reserve.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -require_slug_argument - -# wait for sql to be awake -wait_for_sql - -# check if the site exists -if sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' already exists in bookeeping table. " - echo "Refusing to work" - exit 1; -fi - -log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'" -mkdir -p "$INSTANCE_BASE_DIR" -install_resource_dir "compose/reserve" "$INSTANCE_BASE_DIR" - -log_info " => Writing configuration file" -load_template "docker-env/reserve" \ - "VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \ - "SLUG" "${SLUG}" \ - "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \ - "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \ - > "$INSTANCE_BASE_DIR/.env" - - -log_info " => Running and building image" -cd "$INSTANCE_BASE_DIR" -docker-compose build --pull -docker-compose pull - -log_info " => Starting container" -docker-compose up -d - -log_info " => $INSTANCE_DOMAIN has been reserved" \ No newline at end of file diff --git a/distillery/resources.go b/distillery/resources.go new file mode 100644 index 0000000..9487607 --- /dev/null +++ b/distillery/resources.go @@ -0,0 +1,120 @@ +// TODO: Rename this to resources oncen finished +package distillery + +import ( + "embed" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// resourceEmbed contains all the resources required by the WissKI-Distillery package. +//go:embed all:resources +var resourceEmbed embed.FS + +// InstallResource install a resource src into dest. +// When it encounters a directory, recursively installs the directory is called. +// For each installation item, onInstallFile is called, unless onInstallFile is nil. +// +// If src points to a file, dst must either be an existing file, or not exist. +// If src points to a directory, dst must either be an existing directory, or not exist. +func InstallResource(dst, src string, onInstallFile func(dst, src string)) error { + return installFile(dst, resourceEmbed, src, onInstallFile) +} + +var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory") +var errExpectedDirectoryButGotFile = errors.New("Expected a directory, but got a file") + +func installFile(dst string, fsys embed.FS, src string, onInstallFile func(dst, src string)) error { + // call the on-install file path + if onInstallFile != nil { + onInstallFile(dst, src) + } + + // open the source file! + srcFile, err := fsys.Open(src) + if err != nil { + return errors.Wrapf(err, "Error opening source file %s", src) + } + defer srcFile.Close() + + // stat the source file to install + srcStat, srcErr := srcFile.Stat() + if srcErr != nil { + return errors.Wrapf(srcErr, "Error calling stat on source %s", src) + } + + // if it is a directory, we should recurse! + if srcStat.IsDir() { + return installDir(dst, srcStat, srcFile, fsys, src, onInstallFile) + } + + // determine if we need to create the destination file, or if it already exists + dstStat, dstErr := os.Stat(dst) + flag := os.O_WRONLY + switch { + case os.IsNotExist(dstErr): + flag |= os.O_CREATE + case dstErr != nil: + return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst) + case dstStat.IsDir(): + return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst) + } + + // Open the file + dstFile, err := os.OpenFile(dst, flag, srcStat.Mode()) + if err != nil { + return errors.Wrapf(err, "Error opening destination %s", dst) + } + defer dstFile.Close() + + // copy over the content + _, err = io.Copy(dstFile, srcFile) + return errors.Wrapf(err, "Error writing to destination %s", dst) + +} + +func installDir(dst string, srcStat fs.FileInfo, srcFile fs.File, fsys embed.FS, src string, onInstallFile func(dst, src string)) error { + // make sure it is a directory! + dir, ok := srcFile.(fs.ReadDirFile) + if !ok { + return errExpectedDirectoryButGotFile + } + + // create the destination + dstStat, dstErr := os.Stat(dst) + switch { + case os.IsNotExist(dstErr): + if err := os.MkdirAll(dst, srcStat.Mode()); err != nil { + return errors.Wrapf(err, "Error creating destination directory %s", dst) + } + case dstErr != nil: + return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst) + case !dstStat.IsDir(): + return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst) + case dstErr == nil: + } + + // read the directory + entries, err := dir.ReadDir(-1) + if err != nil { + return errors.Wrapf(err, "Error reading source directory %s", srcFile) + } + + // iterate over all the children + for _, entry := range entries { + if err := func(dst, src string) error { + return installFile(dst, fsys, src, onInstallFile) + }( + filepath.Join(dst, entry.Name()), + filepath.Join(src, entry.Name()), + ); err != nil { + return err + } + } + + return nil +} diff --git a/distillery/resources/compose/barrel/.env.sample b/distillery/resources/compose/barrel/.env.sample index e7517ad..fa89528 100644 --- a/distillery/resources/compose/barrel/.env.sample +++ b/distillery/resources/compose/barrel/.env.sample @@ -3,8 +3,8 @@ ####################### # Real path for volumes to be stored -REAL_PATH=/var/www/example.slug -DISTILLERY_DIR=/distillery/ +REAL_PATH=/var/www/deploy/instances/example.slug +UTILS_DIR=/var/www/deploy/runtime/utils/ ####################### ### Web Server settings diff --git a/distillery/resources/compose/barrel/Dockerfile b/distillery/resources/compose/barrel/Dockerfile index 60dacad..2d9cc39 100644 --- a/distillery/resources/compose/barrel/Dockerfile +++ b/distillery/resources/compose/barrel/Dockerfile @@ -92,7 +92,7 @@ VOLUME /var/www/data # Add and configure the entrypoint ADD scripts/entrypoint.sh /entrypoint.sh -ENTRYPOINT [ "/entrypoint.sh" ] +ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ] CMD ["apache2-foreground"] # Add the provision script and WissKI utils diff --git a/distillery/resources/compose/barrel/docker-compose.yml b/distillery/resources/compose/barrel/docker-compose.yml index 316cabb..a53cf32 100644 --- a/distillery/resources/compose/barrel/docker-compose.yml +++ b/distillery/resources/compose/barrel/docker-compose.yml @@ -25,7 +25,7 @@ services: - ${REAL_PATH}/.composer:/var/www/.composer - ${REAL_PATH}/data:/var/www/data - ${REAL_PATH}/authorized_keys:/var/www/.ssh/authorized_keys - - ${DISTILLERY_DIR}/utils:/utils:ro + - ${UTILS_DIR}:/utils:ro networks: default: diff --git a/distillery/resources/compose/barrel/scripts/provision_container.sh b/distillery/resources/compose/barrel/scripts/provision_container.sh index b661bd8..df6b73c 100644 --- a/distillery/resources/compose/barrel/scripts/provision_container.sh +++ b/distillery/resources/compose/barrel/scripts/provision_container.sh @@ -170,3 +170,5 @@ function printdetails() { echo "Password: $DRUPAL_PASS" } printdetails + +exit 0 \ No newline at end of file diff --git a/distillery/resources/compose/self/.env.sample b/distillery/resources/compose/self/.env.sample deleted file mode 100644 index d44d778..0000000 --- a/distillery/resources/compose/self/.env.sample +++ /dev/null @@ -1,19 +0,0 @@ -####################### -# Meta Settings -####################### -# The target path to redirect to -TARGET=https://github.com/FAU-CDI/wisski-distillery - -# path to .json -SELF_OVERRIDES_FILE=/overrides.json - -####################### -### Web Server settings -####################### -# the hostname for the website -VIRTUAL_HOST=example.com - -# optional letsencrypt support -# when blank, ignore -LETSENCRYPT_HOST= -LETSENCRYPT_EMAIL= diff --git a/distillery/utils/README b/distillery/resources/runtime/utils/README similarity index 100% rename from distillery/utils/README rename to distillery/resources/runtime/utils/README diff --git a/distillery/utils/blind_update.sh b/distillery/resources/runtime/utils/blind_update.sh similarity index 100% rename from distillery/utils/blind_update.sh rename to distillery/resources/runtime/utils/blind_update.sh diff --git a/distillery/utils/create_admin.sh b/distillery/resources/runtime/utils/create_admin.sh similarity index 100% rename from distillery/utils/create_admin.sh rename to distillery/resources/runtime/utils/create_admin.sh diff --git a/distillery/utils/cron.sh b/distillery/resources/runtime/utils/cron.sh similarity index 100% rename from distillery/utils/cron.sh rename to distillery/resources/runtime/utils/cron.sh diff --git a/distillery/utils/install_colorbox.sh b/distillery/resources/runtime/utils/install_colorbox.sh similarity index 100% rename from distillery/utils/install_colorbox.sh rename to distillery/resources/runtime/utils/install_colorbox.sh diff --git a/distillery/utils/patch_easyrdf.sh b/distillery/resources/runtime/utils/patch_easyrdf.sh similarity index 100% rename from distillery/utils/patch_easyrdf.sh rename to distillery/resources/runtime/utils/patch_easyrdf.sh diff --git a/distillery/utils/patch_triples.sh b/distillery/resources/runtime/utils/patch_triples.sh similarity index 100% rename from distillery/utils/patch_triples.sh rename to distillery/resources/runtime/utils/patch_triples.sh diff --git a/distillery/utils/use_wisski.sh b/distillery/resources/runtime/utils/use_wisski.sh similarity index 100% rename from distillery/utils/use_wisski.sh rename to distillery/resources/runtime/utils/use_wisski.sh diff --git a/distillery/utils/wisski_2x_3x.sh b/distillery/resources/runtime/utils/wisski_2x_3x.sh similarity index 100% rename from distillery/utils/wisski_2x_3x.sh rename to distillery/resources/runtime/utils/wisski_2x_3x.sh diff --git a/distillery/.env.sample b/distillery/resources/templates/bootstrap/env similarity index 82% rename from distillery/.env.sample rename to distillery/resources/templates/bootstrap/env index 271bcc8..7484748 100644 --- a/distillery/.env.sample +++ b/distillery/resources/templates/bootstrap/env @@ -1,11 +1,11 @@ # Several docker-compose files are created to manage global services and the system itself. # On top of this all real-system space will be created under this directory. -DEPLOY_ROOT=/var/www/deploy +DEPLOY_ROOT=${DEPLOY_ROOT} # Each created Drupal Instance corresponds to a single domain name. # These domain names should either be a complete domain name or a sub-domain of a default domain. # This setting configures the default domain-name to create subdomains of. -DEFAULT_DOMAIN=localhost.kwarc.info +DEFAULT_DOMAIN=${DEFAULT_DOMAIN} # By default, the default domain redirects to the distillery repository. # If you want to change this, set an alternate domain name here. @@ -17,7 +17,7 @@ SELF_EXTRA_DOMAINS= # You can override individual URLS in the homepage. # Do this by adding URLs (without trailing '/'s) into a JSON file -SELF_OVERRIDES_FILE=/distillery/overrides.json +SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} # The system can support setting up certificate(s) automatically. # It can be enabled by setting an email for certbot certificates. @@ -52,7 +52,12 @@ DISTILLERY_BOOKKEEPING_TABLE=distillery PASSWORD_LENGTH=64 # A file to be used for global authorized_keys for the ssh server. -GLOBAL_AUTHORIZED_KEYS_FILE=/distillery/authorized_keys +GLOBAL_AUTHORIZED_KEYS_FILE=${AUTHORIZED_KEYS_FILE} -# The admin password of the GraphDB interface, to be used for queries -GRAPHDB_ADMIN_PASSWORD=root \ No newline at end of file +# The admin user and password of the GraphDB interface, to be used for queries +GRAPHDB_ADMIN_USER=${GRAPHDB_ADMIN_USER} +GRAPHDB_ADMIN_PASSWORD=${GRAPHDB_ADMIN_PASSWORD} + +# The admin password to use for access to mysql +MYSQL_ADMIN_USER=${MYSQL_ADMIN_USER} +MYSQL_ADMIN_PASSWORD=${MYSQL_ADMIN_PASSWORD} diff --git a/distillery/resources/templates/bootstrap/global_authorized_keys b/distillery/resources/templates/bootstrap/global_authorized_keys new file mode 100644 index 0000000..08826ed --- /dev/null +++ b/distillery/resources/templates/bootstrap/global_authorized_keys @@ -0,0 +1,2 @@ +# This file contains authorized_keys files valid for every repository in the distillery. +# To add a key, add one file per line. diff --git a/distillery/resources/templates/bootstrap/overrides.json b/distillery/resources/templates/bootstrap/overrides.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/distillery/resources/templates/bootstrap/overrides.json @@ -0,0 +1 @@ +{} diff --git a/distillery/resources/templates/docker-env/barrel b/distillery/resources/templates/docker-env/barrel index eea9266..3b77071 100644 --- a/distillery/resources/templates/docker-env/barrel +++ b/distillery/resources/templates/docker-env/barrel @@ -1,5 +1,5 @@ REAL_PATH=${REAL_PATH} -DISTILLERY_DIR=${DISTILLERY_DIR} +UTILS_DIR=${UTILS_DIR} SLUG=${SLUG} VIRTUAL_HOST=${VIRTUAL_HOST} diff --git a/distillery/resources_template.go b/distillery/resources_template.go new file mode 100644 index 0000000..6de7215 --- /dev/null +++ b/distillery/resources_template.go @@ -0,0 +1,112 @@ +package distillery + +import ( + "io" + "io/fs" + "os" + "regexp" + "strings" + + "github.com/pkg/errors" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +var templateRegexp = regexp.MustCompile(`\${[^}]+}`) + +// InstallTemplates open the resource src, and installs it into dst. +// the template resource must fit into memory. +// +// For each variable ${THING} inside dest, a key 'THING' must exist in context. +// Extra or missing template keys are an error. +func InstallTemplate(dst, src string, context map[string]string) error { + bytes, srcMode, err := doTemplate(src, context) + if err != nil { + return err + } + + // determine if we need to create the destination file, or if it already exists + dstStat, dstErr := os.Stat(dst) + flag := os.O_WRONLY + switch { + case os.IsNotExist(dstErr): + flag |= os.O_CREATE + case dstErr != nil: + return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst) + case dstStat.IsDir(): + return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst) + } + + // open and write the destination file + dstFile, err := os.OpenFile(dst, flag, srcMode) + if err != nil { + return errors.Wrapf(err, "Unable to open file %s", dst) + } + _, err = dstFile.Write(bytes) + return errors.Wrapf(err, "Unable to write destination %s", dst) +} + +// ReadTemplate is like InstallTemplate, except that it writes template into a byte slice and returns it. +func ReadTemplate(src string, context map[string]string) ([]byte, error) { + bytes, _, err := doTemplate(src, context) + return bytes, err +} + +func doTemplate(src string, context map[string]string) (bytes []byte, mode fs.FileMode, err error) { + // open the source file! + srcFile, err := resourceEmbed.Open(src) + if err != nil { + return nil, mode, errors.Wrapf(err, "Error opening source file %s", src) + } + defer srcFile.Close() + + // stat the source file to install + srcStat, srcErr := srcFile.Stat() + if srcErr != nil { + return nil, mode, errors.Wrapf(srcErr, "Error calling stat on source %s", src) + } + + // should not be a directory + if srcStat.IsDir() { + return nil, mode, errors.Wrapf(errExpectedFileButGotDirectory, "Error calling stat on source %s", src) + } + + // read the template and replace + templates, err := io.ReadAll(srcFile) + if err != nil { + return nil, mode, errors.Wrapf(err, "Unable to read src file %s", src) + } + + // keep track of context keys that have not been used + unuusedContext := make(map[string]struct{}, len(context)) + for key := range context { + unuusedContext[key] = struct{}{} + } + + // replace the template regexp + // keeping track of unuused errors + var hadError error + templates = templateRegexp.ReplaceAllFunc(templates, func(b []byte) []byte { + name := string(b[2 : len(b)-1]) // remove the leading ${ and trailing } + delete(unuusedContext, name) // mark the key as having been read + + value, ok := context[name] + if hadError != nil && !ok { + hadError = errors.Errorf("key %s missing in context", name) + } + return []byte(value) + }) + + if hadError != nil { + return nil, mode, hadError + } + + if len(unuusedContext) != 0 { + keys := maps.Keys(unuusedContext) + slices.Sort(keys) + return nil, mode, errors.Errorf("additional keys %s in context", strings.Join(keys, ",")) + } + + // return the data and the mode! + return templates, srcStat.Mode(), nil +} diff --git a/distillery/shell.sh b/distillery/shell.sh deleted file mode 100644 index f75a189..0000000 --- a/distillery/shell.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh and read the slug argument. -DISABLE_LOG=1 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" -DISABLE_LOG=0 -require_slug_argument - - -# if the site doesn't exist, I can't open a shell. -if ! sql_bookkeep_exists "$SLUG"; then - log_error "=> Site '$SLUG' does not exist in bookeeping table. " - echo "I can't open a shell there. " - exit 1 -fi; - -# Read everything from the database -read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)" - -# cd into the right directory -cd "$INSTANCE_BASE_DIR" - -# and open a www-data shell -docker-compose exec barrel /user_shell.sh diff --git a/distillery/system_install.sh b/distillery/system_install.sh deleted file mode 100755 index 334a0fa..0000000 --- a/distillery/system_install.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" - -# Read the 'GRAPHDB_ZIP' argument from the command line. -# If it's not set, throw an error. -GRAPHDB_ZIP=$1 -if [ -z "$GRAPHDB_ZIP" ]; then - log_error "Usage: system_install.sh GRAPHDB_ZIP" - exit 1; -fi; - - -# print some general info on the screen -log_info "=> Preparing system to become a WissKI Distillery" -echo "This script will install or upgrade this system to become a WissKI distillery. " -echo "It is idempotent and can safely be run multiple times. " -sleep 5 - - -# Install default system upgrades. -log_info "=> Installing system updates" -apt-get update -apt-get upgrade -y - -# install docker dependencies. -log_info "=> Installing docker installer dependencies" -apt-get update -apt-get install -y curl - -# install docker using an automated script. -log_info "=> Installing docker" -curl -fsSL https://get.docker.com -o - | /bin/sh - -# install docker-compose dependencies. -log_info "=> Install docker-compose installer dependencies" -apt-get update -apt-get install -y python3-pip libffi-dev - -# install docker-compose. -log_info "=> Installing docker-compose" -pip3 install --upgrade docker-compose - -log_info "=> Creating docker-compose directories and files" -mkdir -p "$DEPLOY_INSTANCES_DIR" -mkdir -p "$DEPLOY_WEB_DIR" -mkdir -p "$DEPLOY_SELF_DIR" -mkdir -p "$DEPLOY_RESOLVER_DIR" -mkdir -p "$DEPLOY_SSH_DIR" -mkdir -p "$DEPLOY_TRIPLESTORE_DIR" -mkdir -p "$DEPLOY_SQL_DIR" -mkdir -p "$DEPLOY_BACKUP_INPROGRESS_DIR" -mkdir -p "$DEPLOY_BACKUP_FINAL_DIR" - -log_info "=> Creating 'distillery' network" -docker network create distillery || true - -log_info "=> Creating 'docker-compose' files for the 'web'. " -install_resource_dir "compose/web" "$DEPLOY_WEB_DIR" - -log_info " => Writing 'web' configuration file" -load_template "docker-env/web" \ - "DEFAULT_HOST" "${DEFAULT_DOMAIN}" \ - > "$DEPLOY_WEB_DIR/.env" - -log_info "=> Creating 'docker-compose' files for the 'self'. " -install_resource_dir "compose/self" "$DEPLOY_SELF_DIR" - -log_info "=> Creating 'docker-compose' files for the 'resolver'. " -install_resource_dir "compose/resolver" "$DEPLOY_RESOLVER_DIR" -touch "$DEPLOY_PREFIX_CONFIG" - -log_info "=> Creating 'docker-compose' files for the 'ssh'. " -install_resource_dir "compose/ssh" "$DEPLOY_SSH_DIR" - -# setup the lesencrypt host for the default domain -if [ -n "$LETSENCRYPT_HOST" ]; then - LETSENCRYPT_HOST="$SELF_DOMAIN_SPEC" -fi; - -log_info " => Writing 'self' configuration file" -load_template "docker-env/self" \ - "VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \ - "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \ - "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \ - "TARGET" "${SELF_REDIRECT}" \ - "OVERRIDES_FILE" "${SELF_OVERRIDES_FILE}" \ - > "$DEPLOY_SELF_DIR/.env" - -log_info " => Writing 'resolver' configuration file" -load_template "docker-env/resolver" \ - "VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \ - "LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \ - "LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \ - "PREFIX_FILE" "${DEPLOY_PREFIX_CONFIG}" \ - "DEFAULT_DOMAIN" "${DEFAULT_DOMAIN}" \ - "LEGACY_DOMAIN" "${SELF_EXTRA_DOMAINS}" \ - > "$DEPLOY_RESOLVER_DIR/.env" - -# copy over the directory -log_info "=> Creating 'docker-compose' files for the 'triplestore'. " -install_resource_dir "compose/triplestore" "$DEPLOY_TRIPLESTORE_DIR" - -# copy the graphdb.zip -echo "Writing \"$DEPLOY_TRIPLESTORE_DIR/graphdb.zip\"" -cp "$GRAPHDB_ZIP" "$DEPLOY_TRIPLESTORE_DIR/graphdb.zip" - -# create data (volume) location -mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/data/" -mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/work/" -mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/logs/" - -# copy over the sql resource directory, then ensure the data diretory for sql exists. -log_info "=> Creating 'docker-compose' files for the 'sql'. " -install_resource_dir "compose/sql" "$DEPLOY_SQL_DIR" -mkdir -p "$DEPLOY_SQL_DIR/data/" - -# Run all the updates via system_update.sh -log_info " => Running 'system_update.sh'" -bash "$SCRIPT_DIR/system_update.sh" - -log_info "=> Waiting for sql to come up" -wait_for_sql - -log_info "=> Creating '$DISTILLERY_BOOKKEEPING_DATABASE' database and '$DISTILLERY_BOOKKEEPING_TABLE' table" -load_template "bookkeeping/create.sql" "DATABASE" "$DISTILLERY_BOOKKEEPING_DATABASE" "TABLE" "$DISTILLERY_BOOKKEEPING_TABLE" | \ - dockerized_mysql - -log_info "=> System installation finished, ready to distill. " \ No newline at end of file diff --git a/distillery/system_update.sh b/distillery/system_update.sh deleted file mode 100755 index 44d4cf9..0000000 --- a/distillery/system_update.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -e - -# read the lib/shared.sh -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -cd "$DIR" -source "$DIR/lib/lib.sh" - -log_info "=> Rebuilding and restarting 'web' stack" -update_stack "$DEPLOY_WEB_DIR" - -log_info "=> Rebuilding and restarting 'self' stack" -update_stack "$DEPLOY_SELF_DIR" - -# build and start the ssh server -log_info "=> Rebuilding and restarting 'ssh' stack" -update_stack "$DEPLOY_SSH_DIR" - -# build and start the triplestore -log_info "=> Rebuilding and restarting 'triplestore' stack" -update_stack "$DEPLOY_TRIPLESTORE_DIR" - -# build and start the triplestore -log_info "=> Rebuilding and restarting 'sql' stack" -update_stack "$DEPLOY_SQL_DIR" - -log_info " => Updating Prefix Config" -cd "$DIR" -bash update_prefix_config.sh - -log_info "=> Rebuilding and restarting 'resolver' stack" -update_stack "$DEPLOY_RESOLVER_DIR" - -log_info "=> System up-to-date. " \ No newline at end of file diff --git a/env/dirs.go b/env/dirs.go new file mode 100644 index 0000000..1c7581e --- /dev/null +++ b/env/dirs.go @@ -0,0 +1,23 @@ +package env + +import "path/filepath" + +func (dis Distillery) BackupDir() string { + return filepath.Join(dis.Config.DeployRoot, "backups") +} + +func (dis Distillery) RuntimeDir() string { + return filepath.Join(dis.Config.DeployRoot, "runtime") +} + +func (dis Distillery) RuntimeUtilsDir() string { + return filepath.Join(dis.Config.DeployRoot, "runtime", "utils") +} + +func (dis Distillery) InprogressBackupPath() string { + return filepath.Join(dis.BackupDir(), "inprogress") +} + +func (dis Distillery) FinalBackupPath() string { + return filepath.Join(dis.BackupDir(), "final") +} diff --git a/env/distillery.go b/env/distillery.go new file mode 100644 index 0000000..6045f84 --- /dev/null +++ b/env/distillery.go @@ -0,0 +1,77 @@ +package env + +import ( + "context" + "os" + "strings" + + "github.com/FAU-CDI/wisski-distillery/internal/config" + "github.com/tkw1536/goprogram/exit" +) + +// Distillery represents a running instance for the distillery +type Distillery struct { + Config *config.Config +} + +func (dis Distillery) HTTPSEnabled() bool { + return dis.Config.CertbotEmail != "" +} + +// Returns the default virtual host +func (dis Distillery) DefaultVirtualHost() string { + VIRTUAL_HOST := dis.Config.DefaultDomain + if len(dis.Config.SelfExtraDomains) > 0 { + VIRTUAL_HOST += "," + strings.Join(dis.Config.SelfExtraDomains, ",") + } + return VIRTUAL_HOST +} + +func (dis Distillery) DefaultLetsencryptHost() string { + if !dis.HTTPSEnabled() { + return "" + } + return dis.DefaultVirtualHost() +} + +// Context returns a new Context belonging to this distillery +func (dis Distillery) Context() context.Context { + return context.Background() +} + +var errNoConfigFile = exit.Error{ + ExitCode: exit.ExitGeneralArguments, + Message: "Configuration File does not exist", +} + +var errOpenConfig = exit.Error{ + ExitCode: exit.ExitGeneralArguments, + Message: "error loading configuration file: %s", +} + +// NewDistillery creates a new distillery object from a set of parameters and requirements +func NewDistillery(params Params, req Requirements) (env *Distillery, err error) { + env = &Distillery{} + + // if we don't need to load the config, there is nothing to do + if !req.NeedsConfig { + return + } + + // if there is no no config file, return + cfg := params.ConfigFilePath() + if cfg == "" { + return nil, errNoConfigFile + } + + f, err := os.Open(params.ConfigFilePath()) + if err != nil { + return nil, errOpenConfig.WithMessageF(err) + } + defer f.Close() + + // unmarshal the config + env.Config = &config.Config{} + err = env.Config.Unmarshal(f) + return +} diff --git a/env/instances.go b/env/instances.go new file mode 100644 index 0000000..b87e406 --- /dev/null +++ b/env/instances.go @@ -0,0 +1,359 @@ +package env + +import ( + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/alessio/shellescape" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/stream" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +var errNoBookkeeping = exit.Error{ + Message: "instance %q does not exist in bookkeeping table", + ExitCode: exit.ExitGeneric, +} + +var ErrInstanceNotFound = exit.Error{ + Message: "instance not found", + ExitCode: exit.ExitGeneric, +} + +// Instance returns the instance of the WissKI Distillery with the provided slug +func (dis *Distillery) Instance(slug string) (i Instance, err error) { + if err := dis.SQLWaitForConnection(); err != nil { + return i, err + } + + table, err := dis.sqlBkTable(false) + if err != nil { + return i, err + } + + // find the instance by slug + query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance) + switch { + case query.Error != nil: + return i, errSQL.WithMessageF(query.Error) + case query.RowsAffected == 0: + return i, ErrInstanceNotFound + default: + i.dis = dis + return i, nil + } +} + +// HasInstance checks if the provided instance exists in the bookeeping table +func (dis *Distillery) HasInstance(slug string) (ok bool, err error) { + if err := dis.SQLWaitForConnection(); err != nil { + return false, err + } + + table, err := dis.sqlBkTable(false) + if err != nil { + return false, err + } + + query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok) + if query.Error != nil { + return false, errSQL.WithMessageF(query.Error) + } + return +} + +// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances. +func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) { + if len(slugs) == 0 { + return dis.AllInstances() + } + return dis.InstancesWith(slugs...) +} + +// AllInstances returns all instances of the WissKI Distillery in consistent order. +// +// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order. +func (dis *Distillery) AllInstances() ([]Instance, error) { + return dis.findInstances(true, func(table *gorm.DB) *gorm.DB { + return table + }) +} + +// InstancesWith returns all instances where the slug is in the provided list of names. +// The returned instances are reordered in a consistent order. +func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) { + return dis.findInstances(true, func(table *gorm.DB) *gorm.DB { + return table.Where("slug IN ?", slugs) + }) +} + +// findInstances finds instance objects based on a query in the bookkeeping table +func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) { + if err := dis.SQLWaitForConnection(); err != nil { + return nil, err + } + + // open the bookkeeping table + table, err := dis.sqlBkTable(false) + if err != nil { + return nil, err + } + + // prepare a query + find := table + if order { + find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false}) + } + if query != nil { + find = query(find) + } + + // fetch bookkeeping instances + var bks []bookkeeping.Instance + find = find.Find(&bks) + if find.Error != nil { + return nil, errSQL.WithMessageF(find.Error) + } + + // make proper instances + instances = make([]Instance, len(bks)) + for i, bk := range bks { + instances[i].Instance = bk + instances[i].dis = dis + } + + return instances, nil +} + +// Instance represents a bookkeeping instance +type Instance struct { + bookkeeping.Instance + + // Credentials for the drupal instance + DrupalUsername string + DrupalPassword string + + dis *Distillery +} + +// Update updates the bookkeeping table with this instance. +func (instance *Instance) Update() error { + db, err := instance.dis.sqlBkTable(false) + if err != nil { + return err + } + + // it has never been created => we need to create it in the database + if instance.Instance.Created.IsZero() { + return db.Create(&instance.Instance).Error + } + + // Update based on the primary key! + return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error +} + +// Delete deletes this instance from the bookkeeping table +func (instance *Instance) Delete() error { + db, err := instance.dis.sqlBkTable(false) + if err != nil { + return err + } + + // doesn't exist => nothing to delete + if instance.Instance.Created.IsZero() { + return nil + } + + // delete it directly + return db.Delete(&instance.Instance).Error +} + +// Shell executes a shell command inside the +func (instance Instance) Shell(io stream.IOStream, argv ...string) int { + return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...) +} + +// Domain returns the full domain name of this instance +func (instance Instance) Domain() string { + return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain) +} + +// IfHttps returns value if the distillery has https enabled, the empty string otherwise +// TODO: Fix this to be in a proper place +func (dis *Distillery) IfHttps(value string) string { + if !dis.HTTPSEnabled() { + return "" + } + return value +} + +// URL returns the public URL of this instance +func (instance Instance) URL() *url.URL { + // setup domain and path + url := &url.URL{ + Host: instance.Domain(), + Path: "/", + } + + // use http or https scheme depending on if the distillery has it enabled + if instance.dis.HTTPSEnabled() { + url.Scheme = "https" + } else { + url.Scheme = "http" + } + + return url +} + +// Stack represents a stack representing this instance +func (instance Instance) Stack() stack.Installable { + return stack.Installable{ + Stack: stack.Stack{ + Name: "barrel", + Dir: instance.FilesystemBase, + }, + ContextResource: filepath.Join("resources", "compose", "barrel"), + + EnvFileResource: filepath.Join("resources", "templates", "docker-env", "barrel"), + EnvFileContext: map[string]string{ + "REAL_PATH": instance.FilesystemBase, + + "SLUG": instance.Slug, + "VIRTUAL_HOST": instance.Domain(), + + "LETSENCRYPT_HOST": instance.dis.IfHttps(instance.Domain()), + "LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail), + + "UTILS_DIR": instance.dis.RuntimeUtilsDir(), + "GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile, + }, + + CopyContextFiles: nil, + + TouchFiles: []string{ + "authorized_keys", + }, + + MakeDirsPerm: fs.ModeDir | fs.ModePerm, + MakeDirs: []string{"data", ".composer"}, + } +} + +func (instance Instance) ReserveStack() stack.Installable { + return stack.Installable{ + Stack: stack.Stack{ + Name: "reserve", + Dir: instance.FilesystemBase, + }, + ContextResource: filepath.Join("resources", "compose", "reserve"), + + EnvFileResource: filepath.Join("resources", "templates", "docker-env", "reserve"), + EnvFileContext: map[string]string{ + "VIRTUAL_HOST": instance.Domain(), + + "LETSENCRYPT_HOST": instance.dis.IfHttps(instance.Domain()), + "LETSENCRYPT_EMAIL": instance.dis.IfHttps(instance.dis.Config.CertbotEmail), + }, + } +} + +// Provision provisions an instance, assuming that the required databases already exist. +func (instance Instance) Provision(io stream.IOStream) error { + + // create the basic st! + st := instance.Stack() + if err := st.Install(io, stack.InstallationContext{}); err != nil { + return err + } + + // Pull and build the stack! + if err := st.Update(io, false); err != nil { + return err + } + + provisionParams := []string{ + instance.Domain(), + + instance.SqlDatabase, + instance.SqlUser, + instance.SqlPassword, + + instance.GraphDBRepository, + instance.GraphDBUser, + instance.GraphDBPassword, + + instance.DrupalUsername, + instance.DrupalPassword, + + "", // TODO: DrupalVersion + "", // TODO: WissKIVersion + } + + // escape the parameter + for i, param := range provisionParams { + provisionParams[i] = shellescape.Quote(param) + } + + // figure out the provision script + // TODO: Move the provision script into the control plane! + provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ") + + if st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) != 0 { + return errors.New("Unable to run provision script") + } + + return nil +} + +func (instance *Instance) NoPrefix() bool { + return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip")) +} + +var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") + +// PrefixConfig returns the prefix config belonging to this instance. +func (instance *Instance) PrefixConfig() (config string, err error) { + // if the user requested to skip the prefix, then don't do anything with it! + if instance.NoPrefix() { + return "", nil + } + + var builder strings.Builder + + // domain + builder.WriteString(instance.URL().String() + ":") + builder.WriteString("\n") + + // default prefixes + wu := stream.NewIOStream(&builder, nil, nil, 0) + if instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") != 0 { + return "", errPrefixExecFailed + } + + // custom prefixes + prefixPath := filepath.Join(instance.FilesystemBase, "prefixes") + if fsx.IsFile(prefixPath) { + prefix, err := os.Open(prefixPath) + if err != nil { + return "", err + } + defer prefix.Close() + if _, err := io.Copy(&builder, prefix); err != nil { + return "", err + } + builder.WriteString("\n") + } + + // and done! + return builder.String(), nil +} diff --git a/env/instances_provision.go b/env/instances_provision.go new file mode 100644 index 0000000..1e613c2 --- /dev/null +++ b/env/instances_provision.go @@ -0,0 +1,96 @@ +package env + +import ( + "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" + "github.com/FAU-CDI/wisski-distillery/internal/config" + "github.com/FAU-CDI/wisski-distillery/internal/password" + "github.com/pkg/errors" +) + +func (dis *Distillery) InstancesDir() string { + return filepath.Join(dis.Config.DeployRoot, "instances") +} + +func (dis *Distillery) InstanceDir(slug string) string { + return filepath.Join(dis.InstancesDir(), slug) +} + +func (dis *Distillery) InstanceSQL(slug string) (database, user string) { + database = dis.Config.MysqlDatabasePrefix + slug + user = dis.Config.MysqlUserPrefix + slug + return +} + +func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) { + repo = dis.Config.GraphDBRepoPrefix + slug + user = dis.Config.GraphDBUserPrefix + slug + return +} + +// Password returns a new password +func (dis *Distillery) NewPassword() (value string, err error) { + return password.Password(dis.Config.PasswordLength) +} + +var errInvalidSlug = errors.New("Not a valid slug") + +// NewInstance fills the struct for a new distillery instance. +// It validates that slug is a valid name for an instance. +// +// It does not perform any checks if the instance already exists, or does the creation in the database. +func (dis *Distillery) NewInstance(slug string) (i Instance, err error) { + + // make sure that the slug is valid! + if _, err := config.IsValidSlug(slug); err != nil { + return i, errInvalidSlug + } + + // generate sql data + sqlPassword, err := dis.NewPassword() + if err != nil { + return i, err + } + sqlDB, sqlUser := dis.InstanceSQL(slug) + + // generate ts data + tsPassword, err := dis.NewPassword() + if err != nil { + return i, err + } + tsRepo, tsUser := dis.InstanceGraphDB(slug) + + // generate drupal data + drPassword, err := dis.NewPassword() + if err != nil { + return i, err + } + drUser := "admin" + + // make the instance object! + instance := bookkeeping.Instance{ + Slug: slug, + + OwnerEmail: "", + AutoBlindUpdateEnabled: true, + + FilesystemBase: dis.InstanceDir(slug), + + SqlDatabase: sqlDB, + SqlUser: sqlUser, + SqlPassword: sqlPassword, + + GraphDBRepository: tsRepo, + GraphDBUser: tsUser, + GraphDBPassword: tsPassword, + } + + i.DrupalUsername = drUser + i.DrupalPassword = drPassword + + // store the instance in the object and return it! + i.Instance = instance + i.dis = dis + return i, nil +} diff --git a/env/params.go b/env/params.go new file mode 100644 index 0000000..1c5e4fa --- /dev/null +++ b/env/params.go @@ -0,0 +1,92 @@ +package env + +import ( + "errors" + "os" + "os/user" + "path/filepath" + "strings" + + "github.com/tkw1536/goprogram/exit" +) + +// Params are parameters used for initialization of the environment +type Params struct { + BaseDirectory string +} + +// ConfigFilePath returns the path to the configuration file +func (params Params) ConfigFilePath() string { + if params.BaseDirectory == "" { + return "" + } + return filepath.Join(params.BaseDirectory, ".env") +} + +var errUnableToLoadParams = exit.Error{ + ExitCode: exit.ExitGeneralArguments, + Message: "Unable to configure wdcli environment: %s", +} + +const BaseDirectoryDefault = "/var/www/deploy" + +// ParamsFromEnv creates a new set of parameters from the environment. +// There is no guarantee that the parameters are valid. +func ParamsFromEnv() (params Params, err error) { + // try to read the base directory + value, err := ReadBaseDirectory() + switch { + case os.IsNotExist(err): + params.BaseDirectory = BaseDirectoryDefault + case err == nil: + params.BaseDirectory = value + default: + return params, errUnableToLoadParams.WithMessageF(err) + } + + return params, nil +} + +var baseConfigFile = ".wdcli" + +// ReadBaseDirectory reads the base directory from the environment, or an empty string +func ReadBaseDirectory() (value string, err error) { + // find the current user + usr, err := user.Current() + if err != nil { + return "", err + } + + // read the base config file! + contents, err := os.ReadFile(filepath.Join(usr.HomeDir, baseConfigFile)) + if err != nil { + return "", err + } + + // and trim the spaces! + value = strings.TrimSpace(string(contents)) + + // check that it is actually set! + if len(value) == 0 { + return "", errors.New("ReadBaseDirectory: Directory is empty") + } + + // and return it! + return value, nil +} + +// WriteBaseDirectory writes the base directory to the environment, or returns an error +func WriteBaseDirectory(dir string) error { + // find the current user + usr, err := user.Current() + if err != nil { + return err + } + + // read the base config file! + return os.WriteFile( + filepath.Join(usr.HomeDir, baseConfigFile), + []byte(dir), + os.ModePerm, + ) +} diff --git a/env/requirements.go b/env/requirements.go new file mode 100644 index 0000000..2304537 --- /dev/null +++ b/env/requirements.go @@ -0,0 +1,24 @@ +package env + +import ( + "github.com/tkw1536/goprogram" + "github.com/tkw1536/goprogram/meta" +) + +type Requirements struct { + NeedsConfig bool +} + +// AllowsFlag checks if the provided flag may be passed to fullfill this requirement +// By default it is used only for help page generation, and may be inaccurate. +func (r Requirements) AllowsFlag(flag meta.Flag) bool { + return true +} + +// Validate validates if this requirement is fullfilled for the provided global flags. +// It should return either nil, or an error of type exit.Error. +// +// Validate does not take into account AllowsOption, see ValidateAllowedOptions. +func (r Requirements) Validate(arguments goprogram.Arguments[struct{}]) error { + return nil +} diff --git a/env/stack.go b/env/stack.go new file mode 100644 index 0000000..239ea20 --- /dev/null +++ b/env/stack.go @@ -0,0 +1,30 @@ +package env + +import ( + "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/internal/stack" +) + +// Stacks returns the Stacks of this distillery +func (dis *Distillery) Stacks() []stack.Installable { + // TODO: Do we want to cache these stacks? + return []stack.Installable{ + dis.WebStack(), + dis.SelfStack(), + dis.ResolverStack(), + dis.SSHStack(), + dis.TriplestoreStack(), + dis.SQLStack(), + } +} + +// asCoreStack treats the provided stack as a core component of this distillery. +func (dis *Distillery) asCoreStack(stack stack.Installable) stack.Installable { + stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", stack.Name) + + stack.ContextResource = filepath.Join("resources", "compose", stack.Name) + stack.EnvFileResource = filepath.Join("resources", "templates", "docker-env", stack.Name) + + return stack +} diff --git a/env/stack_resolver.go b/env/stack_resolver.go new file mode 100644 index 0000000..116f75f --- /dev/null +++ b/env/stack_resolver.go @@ -0,0 +1,39 @@ +package env + +import ( + "path/filepath" + "strings" + + "github.com/FAU-CDI/wisski-distillery/internal/stack" +) + +const ResolverPrefixFile = "prefix.cfg" + +func (dis *Distillery) ResolverStack() stack.Installable { + stack := dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "resolver", + }, + + EnvFileContext: map[string]string{ + "VIRTUAL_HOST": dis.DefaultVirtualHost(), + "LETSENCRYPT_HOST": dis.DefaultLetsencryptHost(), + "LETSENCRYPT_EMAIL": dis.Config.CertbotEmail, + "PREFIX_FILE": "", // set below! + "DEFAULT_DOMAIN": dis.Config.DefaultDomain, + "LEGACY_DOMAIN": strings.Join(dis.Config.SelfExtraDomains, ","), + }, + + TouchFiles: []string{ResolverPrefixFile}, + }) + stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, ResolverPrefixFile) + return stack +} + +func (dis *Distillery) ResolverStackPath() string { + return dis.ResolverStack().Dir +} + +func (dis Distillery) ResolverPrefixConfig() string { + return filepath.Join(dis.ResolverStackPath(), ResolverPrefixFile) +} diff --git a/env/stack_self.go b/env/stack_self.go new file mode 100644 index 0000000..241ecad --- /dev/null +++ b/env/stack_self.go @@ -0,0 +1,28 @@ +package env + +import "github.com/FAU-CDI/wisski-distillery/internal/stack" + +func (dis *Distillery) SelfStack() stack.Installable { + TARGET := "https://github.com/FAU-CDI/wisski-distillery" + if dis.Config.SelfRedirect != nil { + TARGET = dis.Config.SelfRedirect.String() + } + + return dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "self", + }, + + EnvFileContext: map[string]string{ + "VIRTUAL_HOST": dis.DefaultVirtualHost(), + "LETSENCRYPT_HOST": dis.DefaultLetsencryptHost(), + "LETSENCRYPT_EMAIL": dis.Config.CertbotEmail, + "TARGET": TARGET, + "OVERRIDES_FILE": dis.Config.SelfOverridesFile, + }, + }) +} + +func (dis *Distillery) SelfStackPath() string { + return dis.SelfStack().Dir +} diff --git a/env/stack_sql.go b/env/stack_sql.go new file mode 100644 index 0000000..5e2b459 --- /dev/null +++ b/env/stack_sql.go @@ -0,0 +1,221 @@ +package env + +import ( + "fmt" + "io/fs" + "time" + + "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/sqle" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/FAU-CDI/wisski-distillery/internal/wait" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/stream" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// SQLStack returns the docker stack that handles the sql database. +func (dis *Distillery) SQLStack() stack.Installable { + return dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "sql", + }, + + MakeDirsPerm: fs.ModeDir | fs.ModePerm, + MakeDirs: []string{ + "data", + }, + }) +} + +// SQLStackPath returns the path the SQLStack() lives at. +func (dis *Distillery) SQLStackPath() string { + return dis.SQLStack().Dir +} + +// sqlOpen opens a new sql connection to the provided database using the administrative credentials +func (env Distillery) sqlOpen(database string, config *gorm.Config) (*gorm.DB, error) { + sql := mysql.Config{ + DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", env.Config.MysqlAdminUser, env.Config.MysqlAdminPassword, "127.0.0.1:3306", database), + DefaultStringSize: 256, + } + + db, err := gorm.Open(mysql.New(sql), config) + if err != nil { + return db, err + } + + gdb, err := db.DB() + if err != nil { + return db, err + } + gdb.SetMaxIdleConns(0) + + return db, nil +} + +var errSQL = exit.Error{ + Message: "error querying sql database: %s", + ExitCode: exit.ExitGeneric, +} + +// sqlBkTable returns a gorm connection to the bookkeeping database. +func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) { + + config := &gorm.Config{} + if silent { + config.Logger = logger.Default.LogMode(logger.Silent) + } + + // open the database + db, err := dis.sqlOpen(dis.Config.DistilleryBookkeepingDatabase, config) + if err != nil { + return nil, errSQL.WithMessageF(err) + } + + // load the table + table := db.Table(dis.Config.DistilleryBookkeepingTable) + if table.Error != nil { + return nil, errSQL.WithMessageF(err) + } + + return table, nil +} + +// SQLShell executes a mysql shell inside the SQLStack. +func (dis *Distillery) SQLShell(io stream.IOStream, argv ...string) int { + return dis.SQLStack().Exec(io, "sql", "mysql", argv...) +} + +var errSQLBootstrap = exit.Error{ + Message: "Unable to boostrap SQL: %s", + ExitCode: exit.ExitGeneric, +} + +const waitSQLInterval = 1 * time.Second + +// SQLWaitForShell waits for the sql database to be reachable via a docker-compose shell +func (dis *Distillery) SQLWaitForShell() error { + n := stream.FromNil() + return wait.Wait(func() bool { + return dis.SQLShell(n, "-e", "show databases;") == 0 + }, waitSQLInterval, dis.Context()) +} + +// SQLWaitForConnection waits for the sql connection to be alive +func (dis *Distillery) SQLWaitForConnection() error { + return wait.Wait(func() bool { + _, err := dis.sqlBkTable(true) + return err == nil + }, waitSQLInterval, dis.Context()) +} + +var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name") + +func (dis *Distillery) sqlRaw(query string, args ...interface{}) bool { + sql := sqle.Format(query, args...) + return dis.SQLShell(stream.FromNil(), "-e", sql) == 0 +} + +// SQLProvision provisions a new sql database and user +func (dis *Distillery) SQLProvision(name, user, password string) error { + // wait for the database + if err := dis.SQLWaitForShell(); err != nil { + return err + } + + // it's not a safe database name! + if !sqle.IsSafeDatabaseName(name) { + return errInvalidDatabaseName + } + + // create the database and user! + if !dis.sqlRaw("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) { + return errors.New("SQLProvision: Failed to create user") + } + + // and done! + return nil +} + +var errSQLPurgeUser = exit.Error{ + Message: "Unable to delete user", + ExitCode: exit.ExitGeneric, +} + +// SQLPurgeUser deletes the specified user from the database +func (dis *Distillery) SQLPurgeUser(user string) error { + if !dis.sqlRaw("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) { + return errSQLPurgeUser + } + + return nil +} + +var errSQLPurgeDB = exit.Error{ + Message: "Unable to delete database", + ExitCode: exit.ExitGeneric, +} + +// SQLPurgeDatabase deletes the specified db from the database +func (dis *Distillery) SQLPurgeDatabase(db string) error { + if !sqle.IsSafeDatabaseName(db) { + return errSQLPurgeDB + } + if !dis.sqlRaw("DROP DATABASE IF EXISTS `" + db + "`") { + return errSQLPurgeDB + } + return nil +} + +// SQLBootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date +func (dis *Distillery) SQLBootstrap(io stream.IOStream) error { + if err := dis.SQLWaitForShell(); err != nil { + return errSQLBootstrap.WithMessageF(err) + } + + // create the admin user + logging.LogMessage(io, "Creating administrative user") + { + username := dis.Config.MysqlAdminUser + password := dis.Config.MysqlAdminPassword + if !dis.sqlRaw("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) { + return errSQLBootstrap.WithMessageF("Unable to create administrative user") + } + } + + // create the admin user + logging.LogMessage(io, "Creating sql database") + { + if !sqle.IsSafeDatabaseName(dis.Config.DistilleryBookkeepingDatabase) { + return errSQLBootstrap.WithMessageF("Unsafe database name") + } + createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", dis.Config.DistilleryBookkeepingDatabase) + if !dis.sqlRaw(createDBSQL) { + return errSQLBootstrap.WithMessageF(createDBSQL) + } + } + + // wait for the database to come up + logging.LogMessage(io, "Waiting for database update to be complete") + dis.SQLWaitForConnection() + + // open the database + logging.LogMessage(io, "Migrating bookkeeping table") + { + db, err := dis.sqlBkTable(false) + if err != nil { + return errSQLBootstrap.WithMessageF(err) + } + + if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil { + return errSQLBootstrap.WithMessageF(err) + } + } + + return nil +} diff --git a/env/stack_ssh.go b/env/stack_ssh.go new file mode 100644 index 0000000..965b870 --- /dev/null +++ b/env/stack_ssh.go @@ -0,0 +1,16 @@ +package env + +import "github.com/FAU-CDI/wisski-distillery/internal/stack" + +func (dis *Distillery) SSHStack() stack.Installable { + // TODO: Ensure that .env is copied if needed + return dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "sql", + }, + }) +} + +func (dis *Distillery) SSHStackPath() string { + return dis.SSHStack().Dir +} diff --git a/env/stack_triplestore.go b/env/stack_triplestore.go new file mode 100644 index 0000000..d7cb436 --- /dev/null +++ b/env/stack_triplestore.go @@ -0,0 +1,263 @@ +package env + +import ( + "bytes" + "encoding/json" + "io" + "io/fs" + "mime/multipart" + "net/http" + "path/filepath" + "time" + + "github.com/FAU-CDI/wisski-distillery/distillery" + "github.com/FAU-CDI/wisski-distillery/internal/logging" + "github.com/FAU-CDI/wisski-distillery/internal/stack" + "github.com/FAU-CDI/wisski-distillery/internal/wait" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/exit" + "github.com/tkw1536/goprogram/stream" +) + +func (dis *Distillery) TriplestoreStack() stack.Installable { + return dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "triplestore", + }, + + CopyContextFiles: []string{"graphdb.zip"}, + + MakeDirsPerm: fs.ModeDir | fs.ModePerm, + MakeDirs: []string{ + filepath.Join("data", "data"), + filepath.Join("data", "work"), + filepath.Join("data", "logs"), + }, + }) +} + +func (dis *Distillery) TriplestoreStackPath() string { + return dis.TriplestoreStack().Dir +} + +type TriplestoreUserPayload struct { + Password string `json:"password"` + AppSettings TriplestoreUserAppSettings `json:"appSettings"` + GrantedAuthorities []string `json:"grantedAuthorities"` +} +type TriplestoreUserAppSettings struct { + DefaultInference bool `json:"DEFAULT_INFERENCE"` + DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"` + DefaultSameas bool `json:"DEFAULT_SAMEAS"` + IgnoreSharedQueries bool `json:"IGNORE_SHARED_QUERIES"` + ExecuteCount bool `json:"EXECUTE_COUNT"` +} + +var errTriplestoreBootstrap = exit.Error{ + Message: "Unable to bootstrap Triplestore: %s", + ExitCode: exit.ExitGeneric, +} + +const triplestoreBaseURL = "http://127.0.0.1:7200" +const waitTSInterval = 1 * time.Second + +// triplestoreCall makes a request to the triplestore. +// +// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name. +// When bodyName is empty, simply marshal body as application/json +func (dis *Distillery) triplestoreRequest(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) { + var reader io.Reader + + var contentType string + + // for "PUT" and "POST" we setup a body + if method == "PUT" || method == "POST" { + if bodyName != "" { + buffer := &bytes.Buffer{} + writer := multipart.NewWriter(buffer) + contentType = writer.FormDataContentType() + + part, err := writer.CreateFormFile(bodyName, "filename.txt") + if err != nil { + return nil, err + } + io.Copy(part, bytes.NewReader(body.([]byte))) + writer.Close() + reader = buffer + } else { + contentType = "application/json" + mbytes, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(mbytes) + } + } + + // create the request object + req, err := http.NewRequest(method, triplestoreBaseURL+url, reader) + if err != nil { + return nil, err + } + + // Setup configuration! + if accept != "" { + req.Header.Set("Accept", accept) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + req.SetBasicAuth(dis.Config.TriplestoreAdminUser, dis.Config.TriplestoreAdminPassword) + + // and send it + return http.DefaultClient.Do(req) +} + +func (dis *Distillery) TriplestoreWaitForConnection() error { + return wait.Wait(func() bool { + res, err := dis.triplestoreRequest("GET", "/rest/repositories", nil, "", "") + if err != nil { + return false + } + defer res.Body.Close() + return true + }, waitTSInterval, dis.Context()) +} + +var errTripleStoreFailedRepository = exit.Error{ + Message: "Failed to create repository: %s", + ExitCode: exit.ExitGeneric, +} + +func (dis *Distillery) TriplestoreProvision(name, domain, user, password string) error { + if err := dis.TriplestoreWaitForConnection(); err != nil { + return err + } + + // prepare the create repo request + createRepo, err := distillery.ReadTemplate(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), map[string]string{ + "GRAPHDB_REPO": name, + "INSTANCE_DOMAIN": domain, + }) + if err != nil { + return err + } + + // do the create! + { + res, err := dis.triplestoreRequest("POST", "/rest/repositories", createRepo, "config", "") + if err != nil { + return errTripleStoreFailedRepository.WithMessageF(err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusCreated { + return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201") + } + } + + // create the user and grant them access + { + res, err := dis.triplestoreRequest("POST", "/rest/security/users/"+user, TriplestoreUserPayload{ + Password: password, + AppSettings: TriplestoreUserAppSettings{ + DefaultInference: true, + DefaultVisGraphSchema: true, + DefaultSameas: true, + IgnoreSharedQueries: false, + ExecuteCount: true, + }, + GrantedAuthorities: []string{ + "ROLE_USER", + "READ_REPO_" + name, + "WRITE_REPO_" + name, + }, + }, "", "") + if err != nil { + return errTripleStoreFailedRepository.WithMessageF(err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusCreated { + return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201") + } + } + + return nil +} + +// TriplestorePurgeUser deletes the specified user from the triplestore +func (dis *Distillery) TriplestorePurgeUser(user string) error { + res, err := dis.triplestoreRequest("DELETE", "/rest/security/users/"+user, nil, "", "") + if err != nil { + return err + } + if res.StatusCode != http.StatusNoContent { + return errors.Errorf("Delete returned code %d", res.StatusCode) + } + return nil +} + +// TriplestorePurgeRepo deletes the specified repo from the triplestore +func (dis *Distillery) TriplestorePurgeRepo(repo string) error { + res, err := dis.triplestoreRequest("DELETE", "/rest/repositories/"+repo, nil, "", "") + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return errors.Errorf("Delete returned code %d", res.StatusCode) + } + return nil +} + +func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error { + logging.LogMessage(io, "Waiting for Triplestore") + if err := dis.TriplestoreWaitForConnection(); err != nil { + return err + } + + logging.LogMessage(io, "Resetting admin user password") + { + res, err := dis.triplestoreRequest("PUT", "/rest/security/users/"+dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{ + Password: dis.Config.TriplestoreAdminPassword, + AppSettings: TriplestoreUserAppSettings{ + DefaultInference: true, + DefaultVisGraphSchema: true, + DefaultSameas: true, + IgnoreSharedQueries: false, + ExecuteCount: true, + }, + GrantedAuthorities: []string{"ROLE_ADMIN"}, + }, "", "") + if err != nil { + return errTriplestoreBootstrap.WithMessageF(err) + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusOK: + // we set the password => requests are unauthorized + // so we still need to enable security (see below!) + case http.StatusUnauthorized: + // a password is needed => security is already enabled. + // the password may or may not work, but that's a problem for later + logging.LogMessage(io, "Security is already enabled") + return nil + default: + return errTriplestoreBootstrap.WithMessageF("Unable to set administrative password") + } + } + + logging.LogMessage(io, "Enabling Triplestore security") + { + res, err := dis.triplestoreRequest("POST", "/rest/security", true, "", "") + if err != nil { + return errTriplestoreBootstrap.WithMessageF(err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return errTriplestoreBootstrap.WithMessageF("Unable to enable security") + } + + return nil + } +} diff --git a/env/stack_web.go b/env/stack_web.go new file mode 100644 index 0000000..ab75427 --- /dev/null +++ b/env/stack_web.go @@ -0,0 +1,19 @@ +package env + +import "github.com/FAU-CDI/wisski-distillery/internal/stack" + +func (dis *Distillery) WebStack() stack.Installable { + return dis.asCoreStack(stack.Installable{ + Stack: stack.Stack{ + Name: "web", + }, + + EnvFileContext: map[string]string{ + "DEFAULT_HOST": dis.Config.DefaultDomain, + }, + }) +} + +func (dis *Distillery) WebStackPath() string { + return dis.WebStack().Dir +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dd2408e --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module github.com/FAU-CDI/wisski-distillery + +go 1.18 + +require ( + github.com/Showmax/go-fqdn v1.0.0 + github.com/alessio/shellescape v1.4.1 + github.com/feiin/sqlstring v0.3.0 + github.com/pkg/errors v0.9.1 + github.com/tkw1536/goprogram v0.0.9 + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e + gorm.io/driver/mysql v1.3.6 + gorm.io/gorm v1.23.8 +) + +require ( + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/jessevdk/go-flags v1.5.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect + golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8fb8663 --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= +github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/feiin/sqlstring v0.3.0 h1:iyPEFijI2BxpY2M+AuhIvdNManzXa2OwGzuPaEMLUgo= +github.com/feiin/sqlstring v0.3.0/go.mod h1:xpZTjVUw1nD3hMgF9SMRdPiooKSikLf4PS5j2NTn3RI= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/tkw1536/goprogram v0.0.9 h1:y5bAWbiVRc47TjvpVDmyMtp5CgJXz1ultLOq+v9tfsA= +github.com/tkw1536/goprogram v0.0.9/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM= +gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= +gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= diff --git a/internal/bookkeeping/bookkeeping.go b/internal/bookkeeping/bookkeeping.go new file mode 100644 index 0000000..690e180 --- /dev/null +++ b/internal/bookkeeping/bookkeeping.go @@ -0,0 +1,68 @@ +// Package bookkeeping implements reading and writing from the bookkeeping table +package bookkeeping + +import ( + "database/sql/driver" + "errors" + "time" +) + +// Instance is a WissKI Instance inside the bookkeeping table. +// It does not represent a running instance; it does not perform any validation. +type Instance struct { + // NOTE: Modifying this struct requires a database migration. + // This should nnever be done unless you know what you're doing. + + // Primary key for the instance + Pk uint `gorm:"column:pk;primaryKey"` + + // time the instance was created + Created time.Time `gorm:"column:created;autoCreateTime"` + + // slug of the system + Slug string `gorm:"column:slug;not null;unique"` + + // email address of the system owner (if any) + OwnerEmail string `gorm:"column:owner_email;type:varchar(320)"` + + // should we automatically enable updates for the system? + AutoBlindUpdateEnabled SQLBit1 `gorm:"column:auto_blind_update_enabled;default:1"` + + // The filesystem path the system can be found under + FilesystemBase string `gorm:"column:filesystem_base;not null"` + + // SQL Database credentials for the system + SqlDatabase string `gorm:"column:sql_database;not null"` + SqlUser string `gorm:"column:sql_user;not null"` + SqlPassword string `gorm:"column:sql_password;not null"` + + // GraphDB Repository + GraphDBRepository string `gorm:"column:graphdb_repository;not null"` + GraphDBUser string `gorm:"column:graphdb_user;not null"` + GraphDBPassword string `gorm:"column:graphdb_password;not null"` +} + +func (i Instance) IsBlindUpdateEnabled() bool { + return bool(i.AutoBlindUpdateEnabled) +} + +// SQLBit1 implements a boolean as a BIT(1) +type SQLBit1 bool + +func (sb SQLBit1) Value() (driver.Value, error) { + if sb { + return []byte{1}, nil + } else { + return []byte{0}, nil + } +} + +var errBadBool = errors.New("SQLBit1: Database does not contain Bit(1)") + +func (sb *SQLBit1) Scan(src interface{}) error { + if bytes, ok := src.([]byte); ok && len(bytes) == 1 { + *sb = bytes[0] == 1 + return nil + } + return errBadBool +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..f2067c1 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,153 @@ +// Package config implements reading and validating a WissKIDistillery configuration file. +package config + +import ( + "fmt" + "io" + "net/url" + "reflect" + "strings" + + "github.com/pkg/errors" +) + +// Config represents the configuration of a distillery instance +type Config struct { + // Several docker-compose files are created to manage global services and the system itself. + // On top of this all real-system space will be created under this directory. + DeployRoot string `env:"DEPLOY_ROOT" default:"/var/www/deploy" validator:"is_valid_abspath"` + + // Each created Drupal Instance corresponds to a single domain name. + // These domain names should either be a complete domain name or a sub-domain of a default domain. + // This setting configures the default domain-name to create subdomains of. + DefaultDomain string `env:"DEFAULT_DOMAIN" default:"localhost.kwarc.info" validator:"is_valid_domain"` + + // By default, the default domain redirects to the distillery repository. + // If you want to change this, set an alternate domain name here. + SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"" validator:"is_valid_https_url"` + + // By default, only the 'self' domain above is caught. + // To catch additional domains, add them here (comma seperated) + SelfExtraDomains []string `env:"SELF_EXTRA_DOMAINS" default:"" validator:"is_valid_domains"` + + // You can override individual URLS in the homepage + // Do this by adding URLs (without trailing '/'s) into a JSON file + SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" validator:"is_valid_file"` + + // The system can support setting up certificate(s) automatically. + // It can be enabled by setting an email for certbot certificates. + // This email address can be configured here. + CertbotEmail string `env:"CERTBOT_EMAIL" default:"" validator:"is_valid_email"` + + // Maximum age for backup + MaxBackupAge int `env:"MAX_BACKUP_AGE" default:"" validator:"is_valid_number"` + + // Each Drupal instance requires a corresponding system user, database users and databases. + // These are also set by the appropriate domain name. + // To differentiate them from other users of the system, these names can be prefixed. + // The prefix to use can be configured here. + // When changing these please consider that no system user may exist that has the same name as a mysql user. + // This is a MariaDB restriction. + MysqlUserPrefix string `env:"MYSQL_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"` + MysqlDatabasePrefix string `env:"MYSQL_DATABASE_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"` + GraphDBUserPrefix string `env:"GRAPHDB_USER_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"` + GraphDBRepoPrefix string `env:"GRAPHDB_REPO_PREFIX" default:"mysql-factory-" validator:"is_valid_slug"` + + // In addition to the filesystem the WissKI distillery requires a single SQL table. + // It uses this database to store a list of installed things. + DistilleryBookkeepingDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" validator:"is_valid_slug"` + DistilleryBookkeepingTable string `env:"DISTILLERY_BOOKKEEPING_TABLE" default:"distillery" validator:"is_valid_slug"` + + // Various components use password-based-authentication. + // These passwords are generated automatically. + // This variable can be used to determine their length. + PasswordLength int `env:"PASSWORD_LENGTH" default:"64" validator:"is_valid_number"` + + // A file to be used for global authorized_keys for the ssh server. + GlobalAuthorizedKeysFile string `env:"GLOBAL_AUTHORIZED_KEYS_FILE" default:"/distillery/authorized_keys" validator:"is_valid_file"` + + // admin credentials for graphdb + TriplestoreAdminUser string `env:"GRAPHDB_ADMIN_USER" default:"admin" validator:"is_nonempty"` + TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD" default:"" validator:"is_nonempty"` + + // admin credentials for the Mysql database + MysqlAdminUser string `env:"MYSQL_ADMIN_USER" default:"admin" validator:"is_nonempty"` + MysqlAdminPassword string `env:"MYSQL_ADMIN_PASSWORD" default:"admin" validator:"is_nonempty"` +} + +func (config Config) String() string { + values := &strings.Builder{} + + vConfig := reflect.ValueOf(config) + tConfig := vConfig.Type() + + // iterate over the types + numValues := tConfig.NumField() + for i := 0; i < numValues; i++ { + tField := tConfig.Field(i) + vField := vConfig.FieldByName(tField.Name) + + fmt.Fprintf(values, "%s=%v\n", tField.Tag.Get("env"), vField.Interface()) + } + + return values.String() +} + +func (config *Config) Unmarshal(src io.Reader) error { + // read all the values! + values, err := ReadAll(src) + if err != nil { + return err + } + + vConfig := reflect.ValueOf(config).Elem() + tConfig := vConfig.Type() + + // iterate over the types + numValues := tConfig.NumField() + for i := 0; i < numValues; i++ { + tField := tConfig.Field(i) + vField := vConfig.FieldByName(tField.Name) + + env := tField.Tag.Get("env") + dflt := tField.Tag.Get("default") + validator := tField.Tag.Get("validator") + + // read the value with a default + value, ok := values[env] + if !ok || value == "" { + if dflt == "" { + continue + } + value = dflt + } + + // use the validator + vFunc, ok := knownValidators[validator] + if vFunc == nil || !ok { + return errors.Errorf("Unable to read %q refers to unknown validator %s", env, validator) + } + + // get the parsed value + checked, err := vFunc(value) + if err != nil { + return errors.Wrapf(err, "Unable to read %q: Validator %s", env, validator) + } + + // set the value of the field + var errSet interface{} + func() { + defer func() { + errSet = recover() + }() + vField.Set(reflect.ValueOf(checked)) + }() + + // capture any error + if errSet != nil { + return errors.Errorf("Unable to parse %q: validator %s returned %q", tField.Name, validator, errSet) + } + } + + return nil +} diff --git a/internal/config/file.go b/internal/config/file.go new file mode 100644 index 0000000..4b29cc7 --- /dev/null +++ b/internal/config/file.go @@ -0,0 +1,76 @@ +package config + +import ( + "bufio" + "io" + "strings" +) + +// Scanner scans an io.Reader for a source file +type Scanner struct { + src *bufio.Scanner + + key string + value string +} + +func NewScanner(r io.Reader) *Scanner { + return &Scanner{ + src: bufio.NewScanner(r), + } +} + +// Scanner advances the scanner to the next variable +func (scanner *Scanner) Scan() bool { + for scanner.src.Scan() { + // check that we don't have an empty or comment only line + tokens := strings.TrimSpace(scanner.src.Text()) + if len(tokens) == 0 || tokens[0] == '#' || strings.HasPrefix(tokens, "//") { + continue + } + + // check that we have a 'key=value' pair + values := strings.SplitN(tokens, "=", 2) + if len(values) != 2 { + continue + } + + // got a key = value + scanner.key = strings.TrimSpace(values[0]) + scanner.value = strings.TrimSpace(values[1]) + return true + } + scanner.key = "" + scanner.value = "" + return false +} + +// Data reads the current value from the scanner. +// When Scan() has not been called, or returned false, returns two empty strings. +func (scanner Scanner) Data() (key, value string) { + return scanner.key, scanner.value +} + +// Error returns an error (if any) +func (scanner Scanner) Error() error { + return scanner.src.Err() +} + +// ReadAll reads all key-value pairs from r. +// If a key occurs more than once, a later occurance overwrites a previous one. +func ReadAll(r io.Reader) (values map[string]string, err error) { + scanner := NewScanner(r) + + // read and store all values + values = make(map[string]string) + for scanner.Scan() { + key, value := scanner.Data() + values[key] = value + } + + // check if there was an error! + if err := scanner.Error(); err != nil { + return nil, err + } + return values, nil +} diff --git a/internal/config/validators.go b/internal/config/validators.go new file mode 100644 index 0000000..f3397af --- /dev/null +++ b/internal/config/validators.go @@ -0,0 +1,105 @@ +package config + +import ( + "net/url" + "regexp" + "strconv" + "strings" + + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/pkg/errors" +) + +// Validator reads from the configuration file +type Validator func(s string) (interface{}, error) + +var knownValidators map[string]Validator = map[string]Validator{ + "is_valid_abspath": IsValidAbspath, + "is_valid_domain": IsValidDomain, + "is_valid_domains": IsValidDomains, + "is_valid_number": IsValidNumber, + "is_valid_https_url": IsValidHttpsURL, + "is_valid_slug": IsValidSlug, + "is_valid_file": IsValidFile, + "is_valid_email": IsValidEmail, + "is_nonempty": IsNonEmpty, +} + +func IsValidAbspath(s string) (interface{}, error) { + if !fsx.IsDirectory(s) { + return nil, errors.Errorf("%q does not exist or is not a directory", s) + } + return s, nil +} + +func IsValidFile(s string) (interface{}, error) { + if !fsx.IsFile(s) { + return nil, errors.Errorf("%q does not exist or is not a regular file", s) + } + return s, nil +} + +func IsNonEmpty(s string) (interface{}, error) { + if s == "" { + return nil, errors.New("value is empty") + } + return s, nil +} + +var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! + +func IsValidDomain(s string) (interface{}, error) { + if !regexpDomain.MatchString(s) { + return nil, errors.Errorf("%q is not a valid domain", s) + } + return s, nil +} +func IsValidDomains(s string) (interface{}, error) { + if len(s) == 0 { + return []string{}, nil + } + domains := strings.Split(s, ",") + for _, d := range domains { + if !regexpDomain.MatchString(d) { + return nil, errors.Errorf("%q is not a valid domain", d) + } + } + return domains, nil +} + +func IsValidNumber(s string) (interface{}, error) { + value, err := strconv.ParseInt(s, 10, 64) + return int(value), err +} + +func IsValidHttpsURL(s string) (interface{}, error) { + url, err := url.Parse(s) + if err != nil { + return nil, errors.Wrapf(err, "%q is not a valid URL", s) + } + if url.Scheme != "https" { + return nil, errors.Errorf("%q is not a valid https URL (%q)", s, url.Scheme) + } + return url, nil +} + +var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! + +func IsValidEmail(s string) (interface{}, error) { + if s == "" { // no email provided + return "", nil + } + if !regexpEmail.MatchString(s) { + return nil, errors.Errorf("%q is not a valid email", s) + } + return s, nil +} + +var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer! + +func IsValidSlug(s string) (interface{}, error) { + if !regexpSlug.MatchString(s) { + return nil, errors.Errorf("%q is not a valid slug", s) + } + return s, nil +} diff --git a/internal/docs.go b/internal/docs.go new file mode 100644 index 0000000..fff05fc --- /dev/null +++ b/internal/docs.go @@ -0,0 +1,4 @@ +// Package internal contains various utility functions. +// +// These are not subject to version guarantees and may be changed +package internal diff --git a/internal/execx/compose.go b/internal/execx/compose.go new file mode 100644 index 0000000..1601b20 --- /dev/null +++ b/internal/execx/compose.go @@ -0,0 +1,11 @@ +package execx + +import ( + "github.com/tkw1536/goprogram/stream" +) + +// Compose runs a docker-compose command in a specific directory, with the provided arguments and streams. +// It then waits for the process to exit, and returns the exit code. +func Compose(io stream.IOStream, workdir string, args ...string) int { + return Exec(io, workdir, "docker", append([]string{"compose"}, args...)...) +} diff --git a/internal/execx/exec.go b/internal/execx/exec.go new file mode 100644 index 0000000..c9bb760 --- /dev/null +++ b/internal/execx/exec.go @@ -0,0 +1,46 @@ +// Package execx defines extensions to the "os/exec" package +package execx + +import ( + "os/exec" + + "github.com/tkw1536/goprogram/stream" +) + +// ExecCommandError is returned by Exec when a command could not be executed. +// This typically hints that the executable cannot be found, but may have other causes. +const ExecCommandError = 127 + +// Exec executes a system command with the specified input/output streams, working directory, and arguments. +// +// If the command executes, it's exit code will be returned. +// If the command can not be executed, returns [ExecCommandError]. +func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { + // setup the command + cmd := exec.Command(exe, argv...) + cmd.Dir = workdir + cmd.Stdin = io.Stdin + cmd.Stdout = io.Stdout + cmd.Stderr = io.Stderr + + // run it + err := cmd.Run() + + // non-zero exit + if err, ok := err.(*exec.ExitError); ok { + return err.ExitCode() + } + + // unknown error + if err != nil { + return ExecCommandError + } + + // everything is fine! + return 0 +} + +// MustExec is like Exec, except that it returns true if the command exited successfully, and else false. +func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool { + return Exec(io, workdir, exe, argv...) == 0 +} diff --git a/internal/fsx/copy.go b/internal/fsx/copy.go new file mode 100644 index 0000000..6fd49f2 --- /dev/null +++ b/internal/fsx/copy.go @@ -0,0 +1,41 @@ +package fsx + +import ( + "errors" + "io" + "os" +) + +var ErrCopySameFile = errors.New("src and dst must be different files") + +// CopyFile copies a file from src to dst. +// When dst and src are the same file, returns ErrCopySameFile. +func CopyFile(dst, src string) error { + if src == dst { + return ErrCopySameFile + } + + // open the source + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + // stat it to get the mode! + srcStat, err := srcFile.Stat() + if err != nil { + return err + } + + // open or create the destination + dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + // and do the copy! + _, err = io.Copy(dstFile, srcFile) + return err +} diff --git a/internal/fsx/touch.go b/internal/fsx/touch.go new file mode 100644 index 0000000..b8cad40 --- /dev/null +++ b/internal/fsx/touch.go @@ -0,0 +1,25 @@ +package fsx + +import ( + "os" + "time" +) + +// Touch touches a file +func Touch(path string) error { + _, err := os.Stat(path) + switch { + case os.IsNotExist(err): + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + return nil + case err != nil: + return err + default: + now := time.Now().Local() + return os.Chtimes(path, now, now) + } +} diff --git a/internal/fsx/type.go b/internal/fsx/type.go new file mode 100644 index 0000000..d4c7c72 --- /dev/null +++ b/internal/fsx/type.go @@ -0,0 +1,13 @@ +package fsx + +import "os" + +func IsDirectory(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsDir() +} + +func IsFile(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} diff --git a/internal/hostname/hostname.go b/internal/hostname/hostname.go new file mode 100644 index 0000000..a252dde --- /dev/null +++ b/internal/hostname/hostname.go @@ -0,0 +1,32 @@ +// Package hostname provides hostname +package hostname + +import ( + "os" + + "github.com/Showmax/go-fqdn" +) + +// FQDN attempts to return the fully qualified domain name of the host system. +// If an error occurs, may fall back to the empty string. +func FQDN() string { + + // try the hostname function + { + fqdn, err := fqdn.FqdnHostname() + if err == nil { + return fqdn + } + } + + // fallback to os hostname + { + hostname, err := os.Hostname() + if err == nil { + return hostname + } + } + + // use the empty string + return "" +} diff --git a/internal/legal/legal.go b/internal/legal/legal.go new file mode 100644 index 0000000..4da57b9 --- /dev/null +++ b/internal/legal/legal.go @@ -0,0 +1,4 @@ +// Package legal contains legal notices. +package legal + +//go:generate gogenlicense -m diff --git a/internal/legal/legal_notices.go b/internal/legal/legal_notices.go new file mode 100755 index 0000000..825f1d9 --- /dev/null +++ b/internal/legal/legal_notices.go @@ -0,0 +1,616 @@ +package legal + +// =========================================================================================================== +// This file was generated automatically at 15-08-2022 10:00:52 using gogenlicense. +// Do not edit manually, as changes may be overwritten. +// =========================================================================================================== + +// Notices contains legal and license information of external software included in this program. +// These notices consist of a list of dependencies along with their license information. +// This string is intended to be displayed to the enduser on demand. +// +// Even though the value of this variable is fixed at compile time it is omitted from this documentation. +// Instead the list of go modules, along with their licenses, is listed below. +// +// # Go Standard Library +// +// The Go Standard Library is licensed under the Terms of the BSD-3-Clause License. +// See also https://golang.org/LICENSE. +// +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// # Module golang org x sys +// +// The Module golang.org/x/sys is licensed under the Terms of the BSD-3-Clause License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/sys@v0.0.0-20220811171246-fbc7d0a398ab/LICENSE. +// +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// # Module golang org x exp +// +// The Module golang.org/x/exp is licensed under the Terms of the BSD-3-Clause License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/exp@v0.0.0-20220722155223-a9213eeb770e/LICENSE. +// +// Copyright (c) 2009 The Go Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// # Module github com tkw1536 goprogram +// +// The Module github.com/tkw1536/goprogram is licensed under the Terms of the MIT License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/tkw1536/goprogram@v0.0.7/LICENSE. +// +// MIT License +// +// Copyright (c) 2022 Tom Wiesing +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// # Module github com pkg errors +// +// The Module github.com/pkg/errors is licensed under the Terms of the BSD-2-Clause License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/pkg/errors@v0.9.1/LICENSE. +// +// Copyright (c) 2015, Dave Cheney +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// # Module github com jessevdk go flags +// +// The Module github.com/jessevdk/go-flags is licensed under the Terms of the BSD-3-Clause License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/jessevdk/go-flags@v1.5.0/LICENSE. +// +// Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// # Module github com go sql driver mysql +// +// The Module github.com/go-sql-driver/mysql is licensed under the Terms of the MPL-2.0 License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/LICENSE. +// +// Mozilla Public License Version 2.0 +// ================================== +// +// 1. Definitions +// -------------- +// +// 1.1. "Contributor" +// means each individual or legal entity that creates, contributes to +// the creation of, or owns Covered Software. +// +// 1.2. "Contributor Version" +// means the combination of the Contributions of others (if any) used +// by a Contributor and that particular Contributor's Contribution. +// +// 1.3. "Contribution" +// means Covered Software of a particular Contributor. +// +// 1.4. "Covered Software" +// means Source Code Form to which the initial Contributor has attached +// the notice in Exhibit A, the Executable Form of such Source Code +// Form, and Modifications of such Source Code Form, in each case +// including portions thereof. +// +// 1.5. "Incompatible With Secondary Licenses" +// means +// +// (a) that the initial Contributor has attached the notice described +// in Exhibit B to the Covered Software; or +// +// (b) that the Covered Software was made available under the terms of +// version 1.1 or earlier of the License, but not also under the +// terms of a Secondary License. +// +// 1.6. "Executable Form" +// means any form of the work other than Source Code Form. +// +// 1.7. "Larger Work" +// means a work that combines Covered Software with other material, in +// a separate file or files, that is not Covered Software. +// +// 1.8. "License" +// means this document. +// +// 1.9. "Licensable" +// means having the right to grant, to the maximum extent possible, +// whether at the time of the initial grant or subsequently, any and +// all of the rights conveyed by this License. +// +// 1.10. "Modifications" +// means any of the following: +// +// (a) any file in Source Code Form that results from an addition to, +// deletion from, or modification of the contents of Covered +// Software; or +// +// (b) any new file in Source Code Form that contains any Covered +// Software. +// +// 1.11. "Patent Claims" of a Contributor +// means any patent claim(s), including without limitation, method, +// process, and apparatus claims, in any patent Licensable by such +// Contributor that would be infringed, but for the grant of the +// License, by the making, using, selling, offering for sale, having +// made, import, or transfer of either its Contributions or its +// Contributor Version. +// +// 1.12. "Secondary License" +// means either the GNU General Public License, Version 2.0, the GNU +// Lesser General Public License, Version 2.1, the GNU Affero General +// Public License, Version 3.0, or any later versions of those +// licenses. +// +// 1.13. "Source Code Form" +// means the form of the work preferred for making modifications. +// +// 1.14. "You" (or "Your") +// means an individual or a legal entity exercising rights under this +// License. For legal entities, "You" includes any entity that +// controls, is controlled by, or is under common control with You. For +// purposes of this definition, "control" means (a) the power, direct +// or indirect, to cause the direction or management of such entity, +// whether by contract or otherwise, or (b) ownership of more than +// fifty percent (50%) of the outstanding shares or beneficial +// ownership of such entity. +// +// 2. License Grants and Conditions +// -------------------------------- +// +// 2.1. Grants +// +// Each Contributor hereby grants You a world-wide, royalty-free, +// non-exclusive license: +// +// (a) under intellectual property rights (other than patent or trademark) +// Licensable by such Contributor to use, reproduce, make available, +// modify, display, perform, distribute, and otherwise exploit its +// Contributions, either on an unmodified basis, with Modifications, or +// as part of a Larger Work; and +// +// (b) under Patent Claims of such Contributor to make, use, sell, offer +// for sale, have made, import, and otherwise transfer either its +// Contributions or its Contributor Version. +// +// 2.2. Effective Date +// +// The licenses granted in Section 2.1 with respect to any Contribution +// become effective for each Contribution on the date the Contributor first +// distributes such Contribution. +// +// 2.3. Limitations on Grant Scope +// +// The licenses granted in this Section 2 are the only rights granted under +// this License. No additional rights or licenses will be implied from the +// distribution or licensing of Covered Software under this License. +// Notwithstanding Section 2.1(b) above, no patent license is granted by a +// Contributor: +// +// (a) for any code that a Contributor has removed from Covered Software; +// or +// +// (b) for infringements caused by: (i) Your and any other third party's +// modifications of Covered Software, or (ii) the combination of its +// Contributions with other software (except as part of its Contributor +// Version); or +// +// (c) under Patent Claims infringed by Covered Software in the absence of +// its Contributions. +// +// This License does not grant any rights in the trademarks, service marks, +// or logos of any Contributor (except as may be necessary to comply with +// the notice requirements in Section 3.4). +// +// 2.4. Subsequent Licenses +// +// No Contributor makes additional grants as a result of Your choice to +// distribute the Covered Software under a subsequent version of this +// License (see Section 10.2) or under the terms of a Secondary License (if +// permitted under the terms of Section 3.3). +// +// 2.5. Representation +// +// Each Contributor represents that the Contributor believes its +// Contributions are its original creation(s) or it has sufficient rights +// to grant the rights to its Contributions conveyed by this License. +// +// 2.6. Fair Use +// +// This License is not intended to limit any rights You have under +// applicable copyright doctrines of fair use, fair dealing, or other +// equivalents. +// +// 2.7. Conditions +// +// Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +// in Section 2.1. +// +// 3. Responsibilities +// ------------------- +// +// 3.1. Distribution of Source Form +// +// All distribution of Covered Software in Source Code Form, including any +// Modifications that You create or to which You contribute, must be under +// the terms of this License. You must inform recipients that the Source +// Code Form of the Covered Software is governed by the terms of this +// License, and how they can obtain a copy of this License. You may not +// attempt to alter or restrict the recipients' rights in the Source Code +// Form. +// +// 3.2. Distribution of Executable Form +// +// If You distribute Covered Software in Executable Form then: +// +// (a) such Covered Software must also be made available in Source Code +// Form, as described in Section 3.1, and You must inform recipients of +// the Executable Form how they can obtain a copy of such Source Code +// Form by reasonable means in a timely manner, at a charge no more +// than the cost of distribution to the recipient; and +// +// (b) You may distribute such Executable Form under the terms of this +// License, or sublicense it under different terms, provided that the +// license for the Executable Form does not attempt to limit or alter +// the recipients' rights in the Source Code Form under this License. +// +// 3.3. Distribution of a Larger Work +// +// You may create and distribute a Larger Work under terms of Your choice, +// provided that You also comply with the requirements of this License for +// the Covered Software. If the Larger Work is a combination of Covered +// Software with a work governed by one or more Secondary Licenses, and the +// Covered Software is not Incompatible With Secondary Licenses, this +// License permits You to additionally distribute such Covered Software +// under the terms of such Secondary License(s), so that the recipient of +// the Larger Work may, at their option, further distribute the Covered +// Software under the terms of either this License or such Secondary +// License(s). +// +// 3.4. Notices +// +// You may not remove or alter the substance of any license notices +// (including copyright notices, patent notices, disclaimers of warranty, +// or limitations of liability) contained within the Source Code Form of +// the Covered Software, except that You may alter any license notices to +// the extent required to remedy known factual inaccuracies. +// +// 3.5. Application of Additional Terms +// +// You may choose to offer, and to charge a fee for, warranty, support, +// indemnity or liability obligations to one or more recipients of Covered +// Software. However, You may do so only on Your own behalf, and not on +// behalf of any Contributor. You must make it absolutely clear that any +// such warranty, support, indemnity, or liability obligation is offered by +// You alone, and You hereby agree to indemnify every Contributor for any +// liability incurred by such Contributor as a result of warranty, support, +// indemnity or liability terms You offer. You may include additional +// disclaimers of warranty and limitations of liability specific to any +// jurisdiction. +// +// 4. Inability to Comply Due to Statute or Regulation +// --------------------------------------------------- +// +// If it is impossible for You to comply with any of the terms of this +// License with respect to some or all of the Covered Software due to +// statute, judicial order, or regulation then You must: (a) comply with +// the terms of this License to the maximum extent possible; and (b) +// describe the limitations and the code they affect. Such description must +// be placed in a text file included with all distributions of the Covered +// Software under this License. Except to the extent prohibited by statute +// or regulation, such description must be sufficiently detailed for a +// recipient of ordinary skill to be able to understand it. +// +// 5. Termination +// -------------- +// +// 5.1. The rights granted under this License will terminate automatically +// if You fail to comply with any of its terms. However, if You become +// compliant, then the rights granted under this License from a particular +// Contributor are reinstated (a) provisionally, unless and until such +// Contributor explicitly and finally terminates Your grants, and (b) on an +// ongoing basis, if such Contributor fails to notify You of the +// non-compliance by some reasonable means prior to 60 days after You have +// come back into compliance. Moreover, Your grants from a particular +// Contributor are reinstated on an ongoing basis if such Contributor +// notifies You of the non-compliance by some reasonable means, this is the +// first time You have received notice of non-compliance with this License +// from such Contributor, and You become compliant prior to 30 days after +// Your receipt of the notice. +// +// 5.2. If You initiate litigation against any entity by asserting a patent +// infringement claim (excluding declaratory judgment actions, +// counter-claims, and cross-claims) alleging that a Contributor Version +// directly or indirectly infringes any patent, then the rights granted to +// You by any and all Contributors for the Covered Software under Section +// 2.1 of this License shall terminate. +// +// 5.3. In the event of termination under Sections 5.1 or 5.2 above, all +// end user license agreements (excluding distributors and resellers) which +// have been validly granted by You or Your distributors under this License +// prior to termination shall survive termination. +// +// ************************************************************************ +// * * +// * 6. Disclaimer of Warranty * +// * ------------------------- * +// * * +// * Covered Software is provided under this License on an "as is" * +// * basis, without warranty of any kind, either expressed, implied, or * +// * statutory, including, without limitation, warranties that the * +// * Covered Software is free of defects, merchantable, fit for a * +// * particular purpose or non-infringing. The entire risk as to the * +// * quality and performance of the Covered Software is with You. * +// * Should any Covered Software prove defective in any respect, You * +// * (not any Contributor) assume the cost of any necessary servicing, * +// * repair, or correction. This disclaimer of warranty constitutes an * +// * essential part of this License. No use of any Covered Software is * +// * authorized under this License except under this disclaimer. * +// * * +// ************************************************************************ +// +// ************************************************************************ +// * * +// * 7. Limitation of Liability * +// * -------------------------- * +// * * +// * Under no circumstances and under no legal theory, whether tort * +// * (including negligence), contract, or otherwise, shall any * +// * Contributor, or anyone who distributes Covered Software as * +// * permitted above, be liable to You for any direct, indirect, * +// * special, incidental, or consequential damages of any character * +// * including, without limitation, damages for lost profits, loss of * +// * goodwill, work stoppage, computer failure or malfunction, or any * +// * and all other commercial damages or losses, even if such party * +// * shall have been informed of the possibility of such damages. This * +// * limitation of liability shall not apply to liability for death or * +// * personal injury resulting from such party's negligence to the * +// * extent applicable law prohibits such limitation. Some * +// * jurisdictions do not allow the exclusion or limitation of * +// * incidental or consequential damages, so this exclusion and * +// * limitation may not apply to You. * +// * * +// ************************************************************************ +// +// 8. Litigation +// ------------- +// +// Any litigation relating to this License may be brought only in the +// courts of a jurisdiction where the defendant maintains its principal +// place of business and such litigation shall be governed by laws of that +// jurisdiction, without reference to its conflict-of-law provisions. +// Nothing in this Section shall prevent a party's ability to bring +// cross-claims or counter-claims. +// +// 9. Miscellaneous +// ---------------- +// +// This License represents the complete agreement concerning the subject +// matter hereof. If any provision of this License is held to be +// unenforceable, such provision shall be reformed only to the extent +// necessary to make it enforceable. Any law or regulation which provides +// that the language of a contract shall be construed against the drafter +// shall not be used to construe this License against a Contributor. +// +// 10. Versions of the License +// --------------------------- +// +// 10.1. New Versions +// +// Mozilla Foundation is the license steward. Except as provided in Section +// 10.3, no one other than the license steward has the right to modify or +// publish new versions of this License. Each version will be given a +// distinguishing version number. +// +// 10.2. Effect of New Versions +// +// You may distribute the Covered Software under the terms of the version +// of the License under which You originally received the Covered Software, +// or under the terms of any subsequent version published by the license +// steward. +// +// 10.3. Modified Versions +// +// If you create software not governed by this License, and you want to +// create a new license for such software, you may create and use a +// modified version of this License if you rename the license and remove +// any references to the name of the license steward (except to note that +// such modified license differs from this License). +// +// 10.4. Distributing Source Code Form that is Incompatible With Secondary +// Licenses +// +// If You choose to distribute Source Code Form that is Incompatible With +// Secondary Licenses under the terms of this version of the License, the +// notice described in Exhibit B of this License must be attached. +// +// Exhibit A - Source Code Form License Notice +// ------------------------------------------- +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// If it is not possible or desirable to put the notice in a particular +// file, then You may include the notice in a location (such as a LICENSE +// file in a relevant directory) where a recipient would be likely to look +// for such a notice. +// +// You may add additional accurate notices of copyright ownership. +// +// Exhibit B - "Incompatible With Secondary Licenses" Notice +// --------------------------------------------------------- +// +// This Source Code Form is "Incompatible With Secondary Licenses", as +// defined by the Mozilla Public License, v. 2.0. +// +// # Module github com alessio shellescape +// +// The Module github.com/alessio/shellescape is licensed under the Terms of the MIT License. +// See also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/alessio/shellescape@v1.4.1/LICENSE. +// +// The MIT License (MIT) +// +// Copyright (c) 2016 Alessio Treglia +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// # Generation +// +// This variable and the associated documentation have been automatically generated using the 'gogenlicense' tool. +// It was last updated at 15-08-2022 10:00:52. +var Notices string + +func init() { + Notices = "The following go packages are imported:\n- Go Standard Library (BSD-3-Clause; see https://golang.org/LICENSE)\n- golang.org/x/sys (BSD-3-Clause; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/sys@v0.0.0-20220811171246-fbc7d0a398ab/LICENSE)\n- golang.org/x/exp (BSD-3-Clause; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/exp@v0.0.0-20220722155223-a9213eeb770e/LICENSE)\n- github.com/tkw1536/goprogram (MIT; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/tkw1536/goprogram@v0.0.7/LICENSE)\n- github.com/pkg/errors (BSD-2-Clause; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/pkg/errors@v0.9.1/LICENSE)\n- github.com/jessevdk/go-flags (BSD-3-Clause; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/jessevdk/go-flags@v1.5.0/LICENSE)\n- github.com/go-sql-driver/mysql (MPL-2.0; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/LICENSE)\n- github.com/alessio/shellescape (MIT; see https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/alessio/shellescape@v1.4.1/LICENSE)\n\n================================================================================\n\n\n================================================================================\nGo Standard Library\nLicensed under the Terms of the BSD-3-Clause License, see also https://golang.org/LICENSE. \n\nCopyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n\t* Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n\t* Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n\t* Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n================================================================================\n\n================================================================================\nModule golang.org/x/sys\nLicensed under the Terms of the BSD-3-Clause License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/sys@v0.0.0-20220811171246-fbc7d0a398ab/LICENSE. \n\nCopyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n================================================================================\n\n================================================================================\nModule golang.org/x/exp\nLicensed under the Terms of the BSD-3-Clause License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/golang.org/x/exp@v0.0.0-20220722155223-a9213eeb770e/LICENSE. \n\nCopyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n================================================================================\n\n================================================================================\nModule github.com/tkw1536/goprogram\nLicensed under the Terms of the MIT License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/tkw1536/goprogram@v0.0.7/LICENSE. \n\nMIT License\n\nCopyright (c) 2022 Tom Wiesing\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n================================================================================\n\n================================================================================\nModule github.com/pkg/errors\nLicensed under the Terms of the BSD-2-Clause License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/pkg/errors@v0.9.1/LICENSE. \n\nCopyright (c) 2015, Dave Cheney \nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n================================================================================\n\n================================================================================\nModule github.com/jessevdk/go-flags\nLicensed under the Terms of the BSD-3-Clause License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/jessevdk/go-flags@v1.5.0/LICENSE. \n\nCopyright (c) 2012 Jesse van den Kieboom. All rights reserved.\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\n copyright notice, this list of conditions and the following disclaimer\n in the documentation and/or other materials provided with the\n distribution.\n * Neither the name of Google Inc. nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n================================================================================\n\n================================================================================\nModule github.com/go-sql-driver/mysql\nLicensed under the Terms of the MPL-2.0 License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/go-sql-driver/mysql@v1.6.0/LICENSE. \n\nMozilla Public License Version 2.0\n==================================\n\n1. Definitions\n--------------\n\n1.1. \"Contributor\"\n means each individual or legal entity that creates, contributes to\n the creation of, or owns Covered Software.\n\n1.2. \"Contributor Version\"\n means the combination of the Contributions of others (if any) used\n by a Contributor and that particular Contributor's Contribution.\n\n1.3. \"Contribution\"\n means Covered Software of a particular Contributor.\n\n1.4. \"Covered Software\"\n means Source Code Form to which the initial Contributor has attached\n the notice in Exhibit A, the Executable Form of such Source Code\n Form, and Modifications of such Source Code Form, in each case\n including portions thereof.\n\n1.5. \"Incompatible With Secondary Licenses\"\n means\n\n (a) that the initial Contributor has attached the notice described\n in Exhibit B to the Covered Software; or\n\n (b) that the Covered Software was made available under the terms of\n version 1.1 or earlier of the License, but not also under the\n terms of a Secondary License.\n\n1.6. \"Executable Form\"\n means any form of the work other than Source Code Form.\n\n1.7. \"Larger Work\"\n means a work that combines Covered Software with other material, in \n a separate file or files, that is not Covered Software.\n\n1.8. \"License\"\n means this document.\n\n1.9. \"Licensable\"\n means having the right to grant, to the maximum extent possible,\n whether at the time of the initial grant or subsequently, any and\n all of the rights conveyed by this License.\n\n1.10. \"Modifications\"\n means any of the following:\n\n (a) any file in Source Code Form that results from an addition to,\n deletion from, or modification of the contents of Covered\n Software; or\n\n (b) any new file in Source Code Form that contains any Covered\n Software.\n\n1.11. \"Patent Claims\" of a Contributor\n means any patent claim(s), including without limitation, method,\n process, and apparatus claims, in any patent Licensable by such\n Contributor that would be infringed, but for the grant of the\n License, by the making, using, selling, offering for sale, having\n made, import, or transfer of either its Contributions or its\n Contributor Version.\n\n1.12. \"Secondary License\"\n means either the GNU General Public License, Version 2.0, the GNU\n Lesser General Public License, Version 2.1, the GNU Affero General\n Public License, Version 3.0, or any later versions of those\n licenses.\n\n1.13. \"Source Code Form\"\n means the form of the work preferred for making modifications.\n\n1.14. \"You\" (or \"Your\")\n means an individual or a legal entity exercising rights under this\n License. For legal entities, \"You\" includes any entity that\n controls, is controlled by, or is under common control with You. For\n purposes of this definition, \"control\" means (a) the power, direct\n or indirect, to cause the direction or management of such entity,\n whether by contract or otherwise, or (b) ownership of more than\n fifty percent (50%) of the outstanding shares or beneficial\n ownership of such entity.\n\n2. License Grants and Conditions\n--------------------------------\n\n2.1. Grants\n\nEach Contributor hereby grants You a world-wide, royalty-free,\nnon-exclusive license:\n\n(a) under intellectual property rights (other than patent or trademark)\n Licensable by such Contributor to use, reproduce, make available,\n modify, display, perform, distribute, and otherwise exploit its\n Contributions, either on an unmodified basis, with Modifications, or\n as part of a Larger Work; and\n\n(b) under Patent Claims of such Contributor to make, use, sell, offer\n for sale, have made, import, and otherwise transfer either its\n Contributions or its Contributor Version.\n\n2.2. Effective Date\n\nThe licenses granted in Section 2.1 with respect to any Contribution\nbecome effective for each Contribution on the date the Contributor first\ndistributes such Contribution.\n\n2.3. Limitations on Grant Scope\n\nThe licenses granted in this Section 2 are the only rights granted under\nthis License. No additional rights or licenses will be implied from the\ndistribution or licensing of Covered Software under this License.\nNotwithstanding Section 2.1(b) above, no patent license is granted by a\nContributor:\n\n(a) for any code that a Contributor has removed from Covered Software;\n or\n\n(b) for infringements caused by: (i) Your and any other third party's\n modifications of Covered Software, or (ii) the combination of its\n Contributions with other software (except as part of its Contributor\n Version); or\n\n(c) under Patent Claims infringed by Covered Software in the absence of\n its Contributions.\n\nThis License does not grant any rights in the trademarks, service marks,\nor logos of any Contributor (except as may be necessary to comply with\nthe notice requirements in Section 3.4).\n\n2.4. Subsequent Licenses\n\nNo Contributor makes additional grants as a result of Your choice to\ndistribute the Covered Software under a subsequent version of this\nLicense (see Section 10.2) or under the terms of a Secondary License (if\npermitted under the terms of Section 3.3).\n\n2.5. Representation\n\nEach Contributor represents that the Contributor believes its\nContributions are its original creation(s) or it has sufficient rights\nto grant the rights to its Contributions conveyed by this License.\n\n2.6. Fair Use\n\nThis License is not intended to limit any rights You have under\napplicable copyright doctrines of fair use, fair dealing, or other\nequivalents.\n\n2.7. Conditions\n\nSections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted\nin Section 2.1.\n\n3. Responsibilities\n-------------------\n\n3.1. Distribution of Source Form\n\nAll distribution of Covered Software in Source Code Form, including any\nModifications that You create or to which You contribute, must be under\nthe terms of this License. You must inform recipients that the Source\nCode Form of the Covered Software is governed by the terms of this\nLicense, and how they can obtain a copy of this License. You may not\nattempt to alter or restrict the recipients' rights in the Source Code\nForm.\n\n3.2. Distribution of Executable Form\n\nIf You distribute Covered Software in Executable Form then:\n\n(a) such Covered Software must also be made available in Source Code\n Form, as described in Section 3.1, and You must inform recipients of\n the Executable Form how they can obtain a copy of such Source Code\n Form by reasonable means in a timely manner, at a charge no more\n than the cost of distribution to the recipient; and\n\n(b) You may distribute such Executable Form under the terms of this\n License, or sublicense it under different terms, provided that the\n license for the Executable Form does not attempt to limit or alter\n the recipients' rights in the Source Code Form under this License.\n\n3.3. Distribution of a Larger Work\n\nYou may create and distribute a Larger Work under terms of Your choice,\nprovided that You also comply with the requirements of this License for\nthe Covered Software. If the Larger Work is a combination of Covered\nSoftware with a work governed by one or more Secondary Licenses, and the\nCovered Software is not Incompatible With Secondary Licenses, this\nLicense permits You to additionally distribute such Covered Software\nunder the terms of such Secondary License(s), so that the recipient of\nthe Larger Work may, at their option, further distribute the Covered\nSoftware under the terms of either this License or such Secondary\nLicense(s).\n\n3.4. Notices\n\nYou may not remove or alter the substance of any license notices\n(including copyright notices, patent notices, disclaimers of warranty,\nor limitations of liability) contained within the Source Code Form of\nthe Covered Software, except that You may alter any license notices to\nthe extent required to remedy known factual inaccuracies.\n\n3.5. Application of Additional Terms\n\nYou may choose to offer, and to charge a fee for, warranty, support,\nindemnity or liability obligations to one or more recipients of Covered\nSoftware. However, You may do so only on Your own behalf, and not on\nbehalf of any Contributor. You must make it absolutely clear that any\nsuch warranty, support, indemnity, or liability obligation is offered by\nYou alone, and You hereby agree to indemnify every Contributor for any\nliability incurred by such Contributor as a result of warranty, support,\nindemnity or liability terms You offer. You may include additional\ndisclaimers of warranty and limitations of liability specific to any\njurisdiction.\n\n4. Inability to Comply Due to Statute or Regulation\n---------------------------------------------------\n\nIf it is impossible for You to comply with any of the terms of this\nLicense with respect to some or all of the Covered Software due to\nstatute, judicial order, or regulation then You must: (a) comply with\nthe terms of this License to the maximum extent possible; and (b)\ndescribe the limitations and the code they affect. Such description must\nbe placed in a text file included with all distributions of the Covered\nSoftware under this License. Except to the extent prohibited by statute\nor regulation, such description must be sufficiently detailed for a\nrecipient of ordinary skill to be able to understand it.\n\n5. Termination\n--------------\n\n5.1. The rights granted under this License will terminate automatically\nif You fail to comply with any of its terms. However, if You become\ncompliant, then the rights granted under this License from a particular\nContributor are reinstated (a) provisionally, unless and until such\nContributor explicitly and finally terminates Your grants, and (b) on an\nongoing basis, if such Contributor fails to notify You of the\nnon-compliance by some reasonable means prior to 60 days after You have\ncome back into compliance. Moreover, Your grants from a particular\nContributor are reinstated on an ongoing basis if such Contributor\nnotifies You of the non-compliance by some reasonable means, this is the\nfirst time You have received notice of non-compliance with this License\nfrom such Contributor, and You become compliant prior to 30 days after\nYour receipt of the notice.\n\n5.2. If You initiate litigation against any entity by asserting a patent\ninfringement claim (excluding declaratory judgment actions,\ncounter-claims, and cross-claims) alleging that a Contributor Version\ndirectly or indirectly infringes any patent, then the rights granted to\nYou by any and all Contributors for the Covered Software under Section\n2.1 of this License shall terminate.\n\n5.3. In the event of termination under Sections 5.1 or 5.2 above, all\nend user license agreements (excluding distributors and resellers) which\nhave been validly granted by You or Your distributors under this License\nprior to termination shall survive termination.\n\n************************************************************************\n* *\n* 6. Disclaimer of Warranty *\n* ------------------------- *\n* *\n* Covered Software is provided under this License on an \"as is\" *\n* basis, without warranty of any kind, either expressed, implied, or *\n* statutory, including, without limitation, warranties that the *\n* Covered Software is free of defects, merchantable, fit for a *\n* particular purpose or non-infringing. The entire risk as to the *\n* quality and performance of the Covered Software is with You. *\n* Should any Covered Software prove defective in any respect, You *\n* (not any Contributor) assume the cost of any necessary servicing, *\n* repair, or correction. This disclaimer of warranty constitutes an *\n* essential part of this License. No use of any Covered Software is *\n* authorized under this License except under this disclaimer. *\n* *\n************************************************************************\n\n************************************************************************\n* *\n* 7. Limitation of Liability *\n* -------------------------- *\n* *\n* Under no circumstances and under no legal theory, whether tort *\n* (including negligence), contract, or otherwise, shall any *\n* Contributor, or anyone who distributes Covered Software as *\n* permitted above, be liable to You for any direct, indirect, *\n* special, incidental, or consequential damages of any character *\n* including, without limitation, damages for lost profits, loss of *\n* goodwill, work stoppage, computer failure or malfunction, or any *\n* and all other commercial damages or losses, even if such party *\n* shall have been informed of the possibility of such damages. This *\n* limitation of liability shall not apply to liability for death or *\n* personal injury resulting from such party's negligence to the *\n* extent applicable law prohibits such limitation. Some *\n* jurisdictions do not allow the exclusion or limitation of *\n* incidental or consequential damages, so this exclusion and *\n* limitation may not apply to You. *\n* *\n************************************************************************\n\n8. Litigation\n-------------\n\nAny litigation relating to this License may be brought only in the\ncourts of a jurisdiction where the defendant maintains its principal\nplace of business and such litigation shall be governed by laws of that\njurisdiction, without reference to its conflict-of-law provisions.\nNothing in this Section shall prevent a party's ability to bring\ncross-claims or counter-claims.\n\n9. Miscellaneous\n----------------\n\nThis License represents the complete agreement concerning the subject\nmatter hereof. If any provision of this License is held to be\nunenforceable, such provision shall be reformed only to the extent\nnecessary to make it enforceable. Any law or regulation which provides\nthat the language of a contract shall be construed against the drafter\nshall not be used to construe this License against a Contributor.\n\n10. Versions of the License\n---------------------------\n\n10.1. New Versions\n\nMozilla Foundation is the license steward. Except as provided in Section\n10.3, no one other than the license steward has the right to modify or\npublish new versions of this License. Each version will be given a\ndistinguishing version number.\n\n10.2. Effect of New Versions\n\nYou may distribute the Covered Software under the terms of the version\nof the License under which You originally received the Covered Software,\nor under the terms of any subsequent version published by the license\nsteward.\n\n10.3. Modified Versions\n\nIf you create software not governed by this License, and you want to\ncreate a new license for such software, you may create and use a\nmodified version of this License if you rename the license and remove\nany references to the name of the license steward (except to note that\nsuch modified license differs from this License).\n\n10.4. Distributing Source Code Form that is Incompatible With Secondary\nLicenses\n\nIf You choose to distribute Source Code Form that is Incompatible With\nSecondary Licenses under the terms of this version of the License, the\nnotice described in Exhibit B of this License must be attached.\n\nExhibit A - Source Code Form License Notice\n-------------------------------------------\n\n This Source Code Form is subject to the terms of the Mozilla Public\n License, v. 2.0. If a copy of the MPL was not distributed with this\n file, You can obtain one at http://mozilla.org/MPL/2.0/.\n\nIf it is not possible or desirable to put the notice in a particular\nfile, then You may include the notice in a location (such as a LICENSE\nfile in a relevant directory) where a recipient would be likely to look\nfor such a notice.\n\nYou may add additional accurate notices of copyright ownership.\n\nExhibit B - \"Incompatible With Secondary Licenses\" Notice\n---------------------------------------------------------\n\n This Source Code Form is \"Incompatible With Secondary Licenses\", as\n defined by the Mozilla Public License, v. 2.0.\n\n================================================================================\n\n================================================================================\nModule github.com/alessio/shellescape\nLicensed under the Terms of the MIT License, see also https://github.com/asdf-vm/asdf/blob/master/installs/golang/1.18.5/packages/pkg/mod/github.com/alessio/shellescape@v1.4.1/LICENSE. \n\nThe MIT License (MIT)\n\nCopyright (c) 2016 Alessio Treglia\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n================================================================================\n" +} diff --git a/internal/logging/level.go b/internal/logging/level.go new file mode 100644 index 0000000..15ae630 --- /dev/null +++ b/internal/logging/level.go @@ -0,0 +1,59 @@ +package logging + +import ( + "sync" + + "github.com/tkw1536/goprogram/stream" +) + +var logLevelMutex sync.Mutex +var logLevelMap = make(map[uintptr]int) + +func getIndent(io stream.IOStream) int { + logLevelMutex.Lock() + defer logLevelMutex.Unlock() + + id, ok := logID(io) + if !ok { + return 0 + } + + return logLevelMap[id] +} + +func incIndent(io stream.IOStream) int { + logLevelMutex.Lock() + defer logLevelMutex.Unlock() + + id, ok := logID(io) + if !ok { // if we don't have an id, then inc statically returns 1 + return 1 + } + + logLevelMap[id]++ + return logLevelMap[id] +} + +func decIndent(io stream.IOStream) int { + logLevelMutex.Lock() + defer logLevelMutex.Unlock() + id, ok := logID(io) + + if !ok { // if we don't have an id, then dec statically returns 0 + return 0 + } + + logLevelMap[id]-- + if logLevelMap[id] < 0 { + panic("DecLogIdent: decrease below 0") + } + return logLevelMap[id] +} + +func logID(io stream.IOStream) (uintptr, bool) { + file, ok := io.Stdin.(interface{ Fd() uintptr }) + if !ok { + return 0, false + } + return file.Fd(), true +} diff --git a/internal/logging/log.go b/internal/logging/log.go new file mode 100644 index 0000000..477ad08 --- /dev/null +++ b/internal/logging/log.go @@ -0,0 +1,30 @@ +package logging + +import ( + "strings" + + "github.com/tkw1536/goprogram/stream" +) + +// LogOperation logs a message that is displayed to the user, and then increases the log indent level. +func LogOperation(operation func() error, io stream.IOStream, format string, args ...interface{}) error { + logOperation(io, getIndent(io), format, args...) + incIndent(io) + defer decIndent(io) + + return operation() +} + +// LogMessage logs a message that is displayed to the user +func LogMessage(io stream.IOStream, format string, args ...interface{}) (int, error) { + return logOperation(io, getIndent(io), format, args...) +} + +func logOperation(io stream.IOStream, indent int, format string, args ...interface{}) (int, error) { + message := "\033[1m" + strings.Repeat(" ", indent+1) + "=> " + format + "\033[0m\n" + if !io.StdinIsATerminal() { + message = " => " + format + } + + return io.Printf(message, args...) +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 0000000..5d2f67c --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,41 @@ +// Package password allows generating random passwords +package password + +import ( + "crypto/rand" + "math/big" + "strings" +) + +// NOTE(twiesing): A bunch of scripts cannot properly handle the extra characters in the password. +// For now it is disabled, but it should be re-enabled later. +const PasswordCharSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // + "!@#$%&*" +const PasswordCharCount = len(PasswordCharSet) + +// Password returns a randomly generated password with the provided length. +// [rand.Reader] is used as the source of randomness. +func Password(length int) (string, error) { + if length < 0 { + panic("length < 0") + } + + var password strings.Builder + password.Grow(length) + + for i := 0; i < length; i++ { + + // grab a random index! + index, err := rand.Int(rand.Reader, big.NewInt(int64(PasswordCharCount))) + if err != nil { + return "", err + } + + // and use that index! + if err := password.WriteByte(PasswordCharSet[int(index.Int64())]); err != nil { + return "", err + } + } + + // return the password! + return password.String(), nil +} diff --git a/internal/sqle/name.go b/internal/sqle/name.go new file mode 100644 index 0000000..53a223f --- /dev/null +++ b/internal/sqle/name.go @@ -0,0 +1,39 @@ +package sqle + +import ( + "strings" + "unicode" +) + +// IsSafeDatabaseName checks if a string is safe to be used as a database name +func IsSafeDatabaseName(value string) bool { + // the empty name is not allowed! + if len(value) == 0 { + return false + } + + // reserved words aren't allowed! + if _, reserved := reservedSQLWords[strings.ToLower(value)]; reserved { + return false + } + + letters := []rune(value) + + // the first letter must be a unicode letter, a @, _ or #. + if !(unicode.IsLetter(letters[0]) || letters[0] == '@' || letters[0] == '_' || letters[0] == '#') { + return false + } + + // each subsequent letter may be a unicode letter, a unicode number, @, _, # or $. + for _, l := range letters[1:] { + if !(unicode.IsLetter(l) || unicode.IsNumber(l) || l == '@' || l == '_' || l == '-' || l == '#' || l == '$') { + return false + } + } + + return true +} + +// reserveredSQLWords is a list of restricted sql words. +var reservedSQLWords = map[string]struct{}{ + "absolute": {}, "action": {}, "ada": {}, "add": {}, "admin": {}, "after": {}, "aggregate": {}, "alias": {}, "all": {}, "allocate": {}, "alter": {}, "and": {}, "any": {}, "are": {}, "array": {}, "as": {}, "asc": {}, "asensitive": {}, "assertion": {}, "asymmetric": {}, "at": {}, "atomic": {}, "authorization": {}, "avg": {}, "backup": {}, "before": {}, "begin": {}, "between": {}, "binary": {}, "bit": {}, "bit_length": {}, "blob": {}, "boolean": {}, "both": {}, "breadth": {}, "break": {}, "browse": {}, "bulk": {}, "by": {}, "call": {}, "called": {}, "cardinality": {}, "cascade": {}, "cascaded": {}, "case": {}, "cast": {}, "catalog": {}, "char": {}, "character": {}, "character_length": {}, "char_length": {}, "check": {}, "checkpoint": {}, "class": {}, "clob": {}, "close": {}, "clustered": {}, "coalesce": {}, "collate": {}, "collation": {}, "collect": {}, "column": {}, "commit": {}, "completion": {}, "compute": {}, "condition": {}, "connect": {}, "connection": {}, "constraint": {}, "constraints": {}, "constructor": {}, "contains": {}, "containstable": {}, "continue": {}, "convert": {}, "corr": {}, "corresponding": {}, "count": {}, "covar_pop": {}, "covar_samp": {}, "create": {}, "cross": {}, "cube": {}, "cume_dist": {}, "current": {}, "current_catalog": {}, "current_date": {}, "current_default_transform_group": {}, "current_path": {}, "current_role": {}, "current_schema": {}, "current_time": {}, "current_timestamp": {}, "current_transform_group_for_type": {}, "current_user": {}, "cursor": {}, "cycle": {}, "data": {}, "database": {}, "date": {}, "day": {}, "dbcc": {}, "deallocate": {}, "dec": {}, "decimal": {}, "declare": {}, "default": {}, "deferrable": {}, "deferred": {}, "delete": {}, "deny": {}, "depth": {}, "deref": {}, "desc": {}, "describe": {}, "descriptor": {}, "destroy": {}, "destructor": {}, "deterministic": {}, "diagnostics": {}, "dictionary": {}, "disconnect": {}, "disk": {}, "distinct": {}, "distributed": {}, "domain": {}, "double": {}, "drop": {}, "dump": {}, "dynamic": {}, "each": {}, "element": {}, "else": {}, "end": {}, "end-exec": {}, "equals": {}, "errlvl": {}, "escape": {}, "every": {}, "except": {}, "exception": {}, "exec": {}, "execute": {}, "exists": {}, "exit": {}, "external": {}, "extract": {}, "false": {}, "fetch": {}, "file": {}, "fillfactor": {}, "filter": {}, "first": {}, "float": {}, "for": {}, "foreign": {}, "fortran": {}, "found": {}, "free": {}, "freetext": {}, "freetexttable": {}, "from": {}, "full": {}, "fulltexttable": {}, "function": {}, "fusion": {}, "general": {}, "get": {}, "global": {}, "go": {}, "goto": {}, "grant": {}, "group": {}, "grouping": {}, "having": {}, "hold": {}, "holdlock": {}, "host": {}, "hour": {}, "identity": {}, "identitycol": {}, "identity_insert": {}, "if": {}, "ignore": {}, "immediate": {}, "in": {}, "include": {}, "index": {}, "indicator": {}, "initialize": {}, "initially": {}, "inner": {}, "inout": {}, "input": {}, "insensitive": {}, "insert": {}, "int": {}, "integer": {}, "intersect": {}, "intersection": {}, "interval": {}, "into": {}, "is": {}, "isolation": {}, "iterate": {}, "join": {}, "key": {}, "kill": {}, "language": {}, "large": {}, "last": {}, "lateral": {}, "leading": {}, "left": {}, "less": {}, "level": {}, "like": {}, "like_regex": {}, "limit": {}, "lineno": {}, "ln": {}, "load": {}, "local": {}, "localtime": {}, "localtimestamp": {}, "locator": {}, "lower": {}, "map": {}, "match": {}, "max": {}, "member": {}, "merge": {}, "method": {}, "min": {}, "minute": {}, "mod": {}, "modifies": {}, "modify": {}, "module": {}, "month": {}, "multiset": {}, "names": {}, "national": {}, "natural": {}, "nchar": {}, "nclob": {}, "new": {}, "next": {}, "no": {}, "nocheck": {}, "nonclustered": {}, "none": {}, "normalize": {}, "not": {}, "null": {}, "nullif": {}, "numeric": {}, "object": {}, "occurrences_regex": {}, "octet_length": {}, "of": {}, "off": {}, "offsets": {}, "old": {}, "on": {}, "only": {}, "open": {}, "opendatasource": {}, "openquery": {}, "openrowset": {}, "openxml": {}, "operation": {}, "option": {}, "or": {}, "order": {}, "ordinality": {}, "out": {}, "outer": {}, "output": {}, "over": {}, "overlaps": {}, "overlay": {}, "pad": {}, "parameter": {}, "parameters": {}, "partial": {}, "partition": {}, "pascal": {}, "path": {}, "percent": {}, "percentile_cont": {}, "percentile_disc": {}, "percent_rank": {}, "pivot": {}, "plan": {}, "position": {}, "position_regex": {}, "postfix": {}, "precision": {}, "prefix": {}, "preorder": {}, "prepare": {}, "preserve": {}, "primary": {}, "print": {}, "prior": {}, "privileges": {}, "proc": {}, "procedure": {}, "public": {}, "raiserror": {}, "range": {}, "read": {}, "reads": {}, "readtext": {}, "real": {}, "reconfigure": {}, "recursive": {}, "ref": {}, "references": {}, "referencing": {}, "regr_avgx": {}, "regr_avgy": {}, "regr_count": {}, "regr_intercept": {}, "regr_r2": {}, "regr_slope": {}, "regr_sxx": {}, "regr_sxy": {}, "regr_syy": {}, "relative": {}, "release": {}, "replication": {}, "restore": {}, "restrict": {}, "result": {}, "return": {}, "returns": {}, "revert": {}, "revoke": {}, "right": {}, "role": {}, "rollback": {}, "rollup": {}, "routine": {}, "row": {}, "rowcount": {}, "rowguidcol": {}, "rows": {}, "rule": {}, "save": {}, "savepoint": {}, "schema": {}, "scope": {}, "scroll": {}, "search": {}, "second": {}, "section": {}, "securityaudit": {}, "select": {}, "sensitive": {}, "sequence": {}, "session": {}, "session_user": {}, "set": {}, "sets": {}, "setuser": {}, "shutdown": {}, "similar": {}, "size": {}, "smallint": {}, "some": {}, "space": {}, "specific": {}, "specifictype": {}, "sql": {}, "sqlca": {}, "sqlcode": {}, "sqlerror": {}, "sqlexception": {}, "sqlstate": {}, "sqlwarning": {}, "start": {}, "state": {}, "statement": {}, "static": {}, "statistics": {}, "stddev_pop": {}, "stddev_samp": {}, "structure": {}, "submultiset": {}, "substring": {}, "substring_regex": {}, "sum": {}, "symmetric": {}, "system": {}, "system_user": {}, "table": {}, "tablesample": {}, "temporary": {}, "terminate": {}, "textsize": {}, "than": {}, "then": {}, "time": {}, "timestamp": {}, "timezone_hour": {}, "timezone_minute": {}, "to": {}, "top": {}, "trailing": {}, "tran": {}, "transaction": {}, "translate": {}, "translate_regex": {}, "translation": {}, "treat": {}, "trigger": {}, "trim": {}, "true": {}, "truncate": {}, "tsequal": {}, "uescape": {}, "under": {}, "union": {}, "unique": {}, "unknown": {}, "unnest": {}, "unpivot": {}, "update": {}, "updatetext": {}, "upper": {}, "usage": {}, "use": {}, "user": {}, "using": {}, "value": {}, "values": {}, "varchar": {}, "variable": {}, "varying": {}, "var_pop": {}, "var_samp": {}, "view": {}, "waitfor": {}, "when": {}, "whenever": {}, "where": {}, "while": {}, "width_bucket": {}, "window": {}, "with": {}, "within": {}, "without": {}, "work": {}, "write": {}, "writetext": {}, "xmlagg": {}, "xmlattributes": {}, "xmlbinary": {}, "xmlcast": {}, "xmlcomment": {}, "xmlconcat": {}, "xmldocument": {}, "xmlelement": {}, "xmlexists": {}, "xmlforest": {}, "xmliterate": {}, "xmlnamespaces": {}, "xmlparse": {}, "xmlpi": {}, "xmlquery": {}, "xmlserialize": {}, "xmltable": {}, "xmltext": {}, "xmlvalidate": {}, "year": {}, "zone": {}} diff --git a/internal/sqle/sqle.go b/internal/sqle/sqle.go new file mode 100644 index 0000000..a8ffd1c --- /dev/null +++ b/internal/sqle/sqle.go @@ -0,0 +1,10 @@ +package sqle + +import ( + "github.com/feiin/sqlstring" +) + +// Format formats the provided query with the given parameters. +func Format(query string, params ...interface{}) string { + return sqlstring.Format(query, params...) +} diff --git a/internal/stack/installable.go b/internal/stack/installable.go new file mode 100644 index 0000000..0e22d7c --- /dev/null +++ b/internal/stack/installable.go @@ -0,0 +1,108 @@ +package stack + +import ( + "io/fs" + "os" + "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/distillery" + "github.com/FAU-CDI/wisski-distillery/internal/fsx" + "github.com/pkg/errors" + "github.com/tkw1536/goprogram/stream" +) + +// Installable represents a Stack that can be automatically installed from a set of resources +// See the Install() method. +type Installable struct { + Stack + + ContextResource string // Path to the resource containing 'docker compose' context + + EnvFileResource string // Path to the resource containing dynamically generated env file + EnvFileContext map[string]string // Context of variables to replace in the env file + + CopyContextFiles []string // Files to copy from the installation context + + TouchFiles []string // Files to 'touch', i.e. ensure that exist + + MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir + MakeDirs []string // directories to ensure that exist +} + +// InstallationContext is a context to install data in +type InstallationContext map[string]string + +// Install installs or updates this stack into the directory specified by stack.Stack(). +// +// Installation is non-interactive, but will provide debugging output onto io. +// InstallationContext +func (is Installable) Install(io stream.IOStream, context InstallationContext) error { + // setup the base files + if err := distillery.InstallResource( + is.Dir, + is.ContextResource, + func(dst, src string) { + io.Printf("[install] %s\n", dst) + }, + ); err != nil { + return err + } + + // configure .env + envDest := filepath.Join(is.Dir, ".env") + if is.EnvFileResource != "" && is.EnvFileContext != nil { + io.Printf("[config] %s\n", envDest) + if err := distillery.InstallTemplate( + envDest, + is.EnvFileResource, + is.EnvFileContext, + ); err != nil { + return err + } + } + + // make sure that certain files exist + for _, name := range is.MakeDirs { + // find the destination! + dst := filepath.Join(is.Dir, name) + + io.Printf("[make] %s\n", dst) + if is.MakeDirsPerm == fs.FileMode(0) { + is.MakeDirsPerm = fs.ModeDir + } + if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil { + return err + } + } + + // copy files from the context! + for _, name := range is.CopyContextFiles { + // find the source! + src, ok := context[name] + if !ok { + return errors.Errorf("Missing file from context: %s", src) + } + + // find the destination! + dst := filepath.Join(is.Dir, name) + + // copy over file from context + io.Printf("[copy] %s (from %s)\n", dst, src) + if err := fsx.CopyFile(dst, src); err != nil { + return errors.Wrapf(err, "Unable to copy file %s", src) + } + } + + // make sure that certain files exist + for _, name := range is.TouchFiles { + // find the destination! + dst := filepath.Join(is.Dir, name) + + io.Printf("[touch] %s\n", dst) + if err := fsx.Touch(dst); err != nil { + return err + } + } + + return nil +} diff --git a/internal/stack/stack.go b/internal/stack/stack.go new file mode 100644 index 0000000..dedc245 --- /dev/null +++ b/internal/stack/stack.go @@ -0,0 +1,110 @@ +// Package stack implements a docker compose stack +package stack + +import ( + "errors" + + "github.com/FAU-CDI/wisski-distillery/internal/execx" + "github.com/tkw1536/goprogram/stream" +) + +// Stack represents a 'docker compose' stack living in a specific directory +// +// NOTE(twiesing): In the current implementation this requires a 'docker' executable on the system. +// This executable must be capable of the 'docker compose' command. +// In the future the idea is to replace this with a native docker compose client. +type Stack struct { + Name string // Name of this stack, TODO: Do we need this? + Dir string // Directory of this stack +} + +var errStackUpdatePull = errors.New("Stack.Update: Pull returned non-zero exit code") +var errStackUpdateBuild = errors.New("Stack.Update: Build returned non-zero exit code") + +// Update pulls, builds, and then optionally starts this stack. +// This does not have a direct 'docker compose' shell equivalent. +// +// See also Up. +func (ds Stack) Update(io stream.IOStream, start bool) error { + if ds.compose(io, "pull") != 0 { + return errStackUpdatePull + } + if ds.compose(io, "build", "--pull") != 0 { + return errStackUpdateBuild + } + if start { + return ds.Up(io) + } + return nil +} + +var errStackUp = errors.New("Stack.Up: Up returned non-zero exit code") + +// Up creates and starts the containers in this Stack. +// It is equivalent to 'docker compose up -d' on the shell. +func (ds Stack) Up(io stream.IOStream) error { + if ds.compose(io, "up", "-d") != 0 { + return errStackUp + } + return nil +} + +// Exec executes an executable in the provided running service. +// It is equivalent to 'docker compose exec $service $executable $args...'. +// +// It returns the exit code of the process. +func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...string) int { + compose := []string{"exec"} + if io.StdinIsATerminal() { + compose = append(compose, "-ti") + } + compose = append(compose, executable) + compose = append(compose, args...) + return ds.compose(io, compose...) +} + +// Run executes the provided service with the given executable. +// It is equivalent to 'docker compose run [--rm] $service $executable $args...'. +// +// It returns the exit code of the process. +func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string, args ...string) int { + compose := []string{"run"} + if autoRemove { + compose = append(compose, "--rm") + } + if !io.StdinIsATerminal() { + compose = append(compose, "-T") + } + compose = append(compose, command) + compose = append(compose, args...) + return ds.compose(io, compose...) +} + +var errStackRestart = errors.New("Stack.Restart: Restart returned non-zero exit code") + +// Restart restarts all containers in this Stack. +// It is equivalent to 'docker compose restart' on the shell. +func (ds Stack) Restart(io stream.IOStream) error { + if ds.compose(io, "restart") != 0 { + return errStackRestart + } + return nil +} + +var errStackDown = errors.New("Stack.Down: Down returned non-zero exit code") + +// Down stops and removes all containers in this Stack. +// It is equivalent to 'docker compose down -v' on the shell. +func (ds Stack) Down(io stream.IOStream) error { + if ds.compose(io, "down", "-v") != 0 { + return errStackDown + } + return nil +} + +// Compose executes a 'docker compose' command on this stack. +// TODO: This should be removed and replaced by an internal call directly to libcompose. +func (ds Stack) compose(io stream.IOStream, args ...string) int { + // TODO: can we migrate to a built-in version of this? + return execx.Compose(io, ds.Dir, args...) +} diff --git a/internal/wait/wait.go b/internal/wait/wait.go new file mode 100644 index 0000000..5763703 --- /dev/null +++ b/internal/wait/wait.go @@ -0,0 +1,31 @@ +package wait + +import ( + "context" + "time" +) + +// Wait repeatedly invokes f, until it returns true or the context is closed. +// The invocation interval is determined by interval. +func Wait(f func() bool, interval time.Duration, context context.Context) error { + // create a new timer + timer := time.NewTimer(interval) + if !timer.Stop() { + <-timer.C + } + defer timer.Stop() + + for { + if f() { + return nil + } + + // reset the timer, and wait for it again! + timer.Reset(interval) + select { + case <-timer.C: + case <-context.Done(): + return context.Err() + } + } +} diff --git a/license.go b/license.go new file mode 100644 index 0000000..ea998ab --- /dev/null +++ b/license.go @@ -0,0 +1,6 @@ +package wisski_distillery + +import _ "embed" + +//go:embed LICENSE +var License string diff --git a/program.go b/program.go new file mode 100644 index 0000000..ab270ad --- /dev/null +++ b/program.go @@ -0,0 +1,44 @@ +package wisski_distillery + +import ( + "os/user" + + "github.com/FAU-CDI/wisski-distillery/env" + "github.com/tkw1536/goprogram" + "github.com/tkw1536/goprogram/exit" +) + +// these define the ggman-specific program types +// none of these are strictly needed, they're just around for convenience +type wdcliEnv = *env.Distillery +type wdcliParameters = env.Params +type wdcliRequirements = env.Requirements +type wdCliFlags = struct{} + +type Program = goprogram.Program[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements] +type Command = goprogram.Command[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements] +type Context = goprogram.Context[wdcliEnv, wdcliParameters, wdCliFlags, wdcliRequirements] +type Arguments = goprogram.Arguments[wdCliFlags] +type Description = goprogram.Description[wdCliFlags, wdcliRequirements] + +// an error when nor arguments are provided. +var errUserIsNotRoot = exit.Error{ + ExitCode: exit.ExitGeneralArguments, + Message: "This command has to be executed as root. The current user is not root.", +} + +func NewProgram() Program { + return Program{ + BeforeCommand: func(context Context, command Command) error { + usr, err := user.Current() + if err != nil || usr.Uid != "0" || usr.Gid != "0" { // make sure that we are root! + return errUserIsNotRoot + } + return nil + }, + + NewEnvironment: func(params wdcliParameters, context Context) (e wdcliEnv, err error) { + return env.NewDistillery(params, context.Description.Requirements) + }, + } +}