diff --git a/cmd/provision.go b/cmd/provision.go index b6aaa7e..4243b7b 100644 --- a/cmd/provision.go +++ b/cmd/provision.go @@ -107,7 +107,7 @@ func (p provision) Run(context wisski_distillery.Context) error { // start the container! logging.LogMessage(context.IOStream, "Starting Container") - if err := instance.Stack().Up(context.IOStream); err != nil { + if err := instance.Barrel().Up(context.IOStream); err != nil { return err } diff --git a/cmd/purge.go b/cmd/purge.go index 60f61ca..86e61eb 100644 --- a/cmd/purge.go +++ b/cmd/purge.go @@ -67,7 +67,7 @@ func (p purge) Run(context wisski_distillery.Context) error { // remove docker stack logging.LogMessage(context.IOStream, "Stopping and removing docker container") - if err := instance.Stack().Down(context.IOStream); err != nil { + if err := instance.Barrel().Down(context.IOStream); err != nil { context.EPrintln(err) } diff --git a/cmd/rebuild.go b/cmd/rebuild.go index bf90be3..88cb704 100644 --- a/cmd/rebuild.go +++ b/cmd/rebuild.go @@ -42,7 +42,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error { var globalErr error for _, instance := range instances { logging.LogOperation(func() error { - s := instance.Stack() + s := instance.Barrel() if err := logging.LogOperation(func() error { return s.Install(context.IOStream, component.InstallationContext{}) }, context.IOStream, "Installing docker stack"); err != nil { diff --git a/cmd/reserve.go b/cmd/reserve.go index 16a307f..997748d 100644 --- a/cmd/reserve.go +++ b/cmd/reserve.go @@ -63,7 +63,7 @@ func (r reserve) Run(context wisski_distillery.Context) error { } // setup docker stack - s := instance.ReserveStack() + s := instance.Reserve() { if err := logging.LogOperation(func() error { return s.Install(context.IOStream, component.InstallationContext{}) diff --git a/go.work b/go.work new file mode 100644 index 0000000..b7a0838 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.18 + +use ( + . + ../../tkw1536/goprogram +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..1d60d83 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,4 @@ +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/component/instances/wisski_create.go b/internal/component/instances/wisski_create.go index 7ff29df..412c23a 100644 --- a/internal/component/instances/wisski_create.go +++ b/internal/component/instances/wisski_create.go @@ -3,8 +3,12 @@ package instances import ( "errors" "path/filepath" + "strings" + "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/pkg/stringparser" + "github.com/alessio/shellescape" + "github.com/tkw1536/goprogram/stream" ) var errInvalidSlug = errors.New("not a valid slug") @@ -59,3 +63,55 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) { wisski.instances = instances return wisski, nil } + +// Provision provisions an instance, assuming that the required databases already exist. +func (wisski WissKI) Provision(io stream.IOStream) error { + + // create the basic st! + st := wisski.Barrel() + if err := st.Install(io, component.InstallationContext{}); err != nil { + return err + } + + // Pull and build the stack! + if err := st.Update(io, false); err != nil { + return err + } + + provisionParams := []string{ + wisski.Domain(), + + wisski.SqlDatabase, + wisski.SqlUsername, + wisski.SqlPassword, + + wisski.GraphDBRepository, + wisski.GraphDBUsername, + wisski.GraphDBPassword, + + wisski.DrupalUsername, + wisski.DrupalPassword, + + "", // TODO: DrupalVersion + "", // TODO: WissKIVersion + } + + // escape the parameter + for i, param := range provisionParams { + provisionParams[i] = shellescape.Quote(param) + } + + // figure out the provision script + // TODO: Move the provision script into the control plane! + provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ") + + code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) + if err != nil { + return err + } + if code != 0 { + return errors.New("unable to run provision script") + } + + return nil +} diff --git a/internal/component/instances/wisski_db.go b/internal/component/instances/wisski_db.go index cfec213..fcaa1b4 100644 --- a/internal/component/instances/wisski_db.go +++ b/internal/component/instances/wisski_db.go @@ -1,25 +1,7 @@ package instances import ( - "bytes" - "embed" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "net/url" - "os" - "path/filepath" - "strings" - "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" - "github.com/FAU-CDI/wisski-distillery/internal/component" - "github.com/FAU-CDI/wisski-distillery/pkg/fsx" - "github.com/alessio/shellescape" - "github.com/tkw1536/goprogram/stream" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" ) // WissKI represents a single WissKI Instance @@ -66,220 +48,3 @@ func (wisski *WissKI) Delete() error { // delete it directly return db.Delete(&wisski.Instance).Error } - -// Shell executes a shell command inside the -func (wisski WissKI) Shell(io stream.IOStream, argv ...string) (int, error) { - return wisski.Stack().Exec(io, "barrel", "/bin/sh", append([]string{"/user_shell.sh"}, argv...)...) -} - -// Domain returns the full domain name of this instance -func (wisski WissKI) Domain() string { - return fmt.Sprintf("%s.%s", wisski.Slug, wisski.instances.Config.DefaultDomain) -} - -// URL returns the public URL of this instance -func (wisski WissKI) URL() *url.URL { - // setup domain and path - url := &url.URL{ - Host: wisski.Domain(), - Path: "/", - } - - // use http or https scheme depending on if the distillery has it enabled - if wisski.instances.Config.HTTPSEnabled() { - url.Scheme = "https" - } else { - url.Scheme = "http" - } - - return url -} - -//go:embed all:instances/barrel instances/barrel.env -var barrelResources embed.FS - -// Stack represents a stack representing this instance -func (wisski WissKI) Stack() component.Installable { - return component.Installable{ - Stack: component.Stack{ - Dir: wisski.FilesystemBase, - }, - - Resources: barrelResources, - ContextPath: filepath.Join("instances", "barrel"), - EnvPath: filepath.Join("instances", "barrel.env"), - - EnvContext: map[string]string{ - "DATA_PATH": filepath.Join(wisski.FilesystemBase, "data"), - - "SLUG": wisski.Slug, - "VIRTUAL_HOST": wisski.Domain(), - - "LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()), - "LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail), - - "RUNTIME_DIR": wisski.instances.Config.RuntimeDir(), - "GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile, - }, - - MakeDirsPerm: fs.ModeDir | fs.ModePerm, - MakeDirs: []string{"data", ".composer"}, - - TouchFiles: []string{ - filepath.Join("data", "authorized_keys"), - }, - } -} - -//go:embed all:instances/reserve instances/reserve.env -var reserveResources embed.FS - -func (wisski WissKI) ReserveStack() component.Installable { - return component.Installable{ - Stack: component.Stack{ - Dir: wisski.FilesystemBase, - }, - - Resources: reserveResources, - ContextPath: filepath.Join("instances", "reserve"), - EnvPath: filepath.Join("instances", "reserve.env"), - - EnvContext: map[string]string{ - "VIRTUAL_HOST": wisski.Domain(), - - "LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()), - "LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail), - }, - } -} - -// Provision provisions an instance, assuming that the required databases already exist. -func (wisski WissKI) Provision(io stream.IOStream) error { - - // create the basic st! - st := wisski.Stack() - if err := st.Install(io, component.InstallationContext{}); err != nil { - return err - } - - // Pull and build the stack! - if err := st.Update(io, false); err != nil { - return err - } - - provisionParams := []string{ - wisski.Domain(), - - wisski.SqlDatabase, - wisski.SqlUsername, - wisski.SqlPassword, - - wisski.GraphDBRepository, - wisski.GraphDBUsername, - wisski.GraphDBPassword, - - wisski.DrupalUsername, - wisski.DrupalPassword, - - "", // TODO: DrupalVersion - "", // TODO: WissKIVersion - } - - // escape the parameter - for i, param := range provisionParams { - provisionParams[i] = shellescape.Quote(param) - } - - // figure out the provision script - // TODO: Move the provision script into the control plane! - provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ") - - code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) - if err != nil { - return err - } - if code != 0 { - return errors.New("unable to run provision script") - } - - return nil -} - -// NoPrefix checks if this WissKI instance is excluded from generating prefixes -func (wisski *WissKI) NoPrefix() bool { - return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip")) -} - -var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") - -// PrefixConfig returns the prefix config belonging to this instance. -func (wisski *WissKI) PrefixConfig() (config string, err error) { - // if the user requested to skip the prefix, then don't do anything with it! - if wisski.NoPrefix() { - return "", nil - } - - var builder strings.Builder - - // domain - builder.WriteString(wisski.URL().String() + ":") - builder.WriteString("\n") - - // default prefixes - wu := stream.NewIOStream(&builder, nil, nil, 0) - code, err := wisski.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") - if err != nil || code != 0 { - return "", errPrefixExecFailed - } - - // custom prefixes - prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes") - if fsx.IsFile(prefixPath) { - prefix, err := os.Open(prefixPath) - if err != nil { - return "", err - } - defer prefix.Close() - if _, err := io.Copy(&builder, prefix); err != nil { - return "", err - } - builder.WriteString("\n") - } - - // and done! - return builder.String(), nil -} - -var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder") - -// ExportPathbuilders writes pathbuilders into the directory dest -func (wisski *WissKI) ExportPathbuilders(dest string) error { - // export all the pathbuilders into the buffer - var buffer bytes.Buffer - wu := stream.NewIOStream(&buffer, nil, nil, 0) - code, err := wisski.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php") - if err != nil || code != 0 { - return errPathbuildersExecFailed - } - - // decode them as a json array - var pathbuilders map[string]string - if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil { - return err - } - - // sort the names of the pathbuilders - names := maps.Keys(pathbuilders) - slices.Sort(names) - - // write each into a file! - for _, name := range names { - pbxml := []byte(pathbuilders[name]) - name := filepath.Join(dest, fmt.Sprintf("%s.xml", name)) - if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil { - return err - } - } - - return nil -} diff --git a/internal/component/instances/wisski_exec.go b/internal/component/instances/wisski_exec.go new file mode 100644 index 0000000..952cefe --- /dev/null +++ b/internal/component/instances/wisski_exec.go @@ -0,0 +1,8 @@ +package instances + +import "github.com/tkw1536/goprogram/stream" + +// Shell executes a shell command inside the instance. +func (wisski WissKI) Shell(io stream.IOStream, argv ...string) (int, error) { + return wisski.Barrel().Exec(io, "barrel", "/bin/sh", append([]string{"/user_shell.sh"}, argv...)...) +} diff --git a/internal/component/instances/wisski_pathbuilders.go b/internal/component/instances/wisski_pathbuilders.go new file mode 100644 index 0000000..60edcdc --- /dev/null +++ b/internal/component/instances/wisski_pathbuilders.go @@ -0,0 +1,49 @@ +package instances + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/tkw1536/goprogram/stream" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" +) + +var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder") + +// ExportPathbuilders writes pathbuilders into the directory dest +func (wisski *WissKI) ExportPathbuilders(dest string) error { + // export all the pathbuilders into the buffer + var buffer bytes.Buffer + wu := stream.NewIOStream(&buffer, nil, nil, 0) + code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php") + if err != nil || code != 0 { + return errPathbuildersExecFailed + } + + // decode them as a json array + var pathbuilders map[string]string + if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil { + return err + } + + // sort the names of the pathbuilders + names := maps.Keys(pathbuilders) + slices.Sort(names) + + // write each into a file! + for _, name := range names { + pbxml := []byte(pathbuilders[name]) + name := filepath.Join(dest, fmt.Sprintf("%s.xml", name)) + if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil { + return err + } + } + + return nil +} diff --git a/internal/component/instances/wisski_prefix.go b/internal/component/instances/wisski_prefix.go new file mode 100644 index 0000000..ebaef07 --- /dev/null +++ b/internal/component/instances/wisski_prefix.go @@ -0,0 +1,58 @@ +package instances + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/FAU-CDI/wisski-distillery/pkg/fsx" + "github.com/tkw1536/goprogram/stream" +) + +// NoPrefix checks if this WissKI instance is excluded from generating prefixes. +// TODO: Move this to the database! +func (wisski *WissKI) NoPrefix() bool { + return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip")) +} + +var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") + +// PrefixConfig returns the prefix config belonging to this instance. +func (wisski *WissKI) PrefixConfig() (config string, err error) { + // if the user requested to skip the prefix, then don't do anything with it! + if wisski.NoPrefix() { + return "", nil + } + + var builder strings.Builder + + // domain + builder.WriteString(wisski.URL().String() + ":") + builder.WriteString("\n") + + // default prefixes + wu := stream.NewIOStream(&builder, nil, nil, 0) + code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") + if err != nil || code != 0 { + return "", errPrefixExecFailed + } + + // custom prefixes + prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes") + if fsx.IsFile(prefixPath) { + prefix, err := os.Open(prefixPath) + if err != nil { + return "", err + } + defer prefix.Close() + if _, err := io.Copy(&builder, prefix); err != nil { + return "", err + } + builder.WriteString("\n") + } + + // and done! + return builder.String(), nil +} diff --git a/internal/component/instances/wisski_props.go b/internal/component/instances/wisski_props.go new file mode 100644 index 0000000..d488f08 --- /dev/null +++ b/internal/component/instances/wisski_props.go @@ -0,0 +1,29 @@ +package instances + +import ( + "fmt" + "net/url" +) + +// Domain returns the full domain name of this WissKI +func (wisski WissKI) Domain() string { + return fmt.Sprintf("%s.%s", wisski.Slug, wisski.instances.Config.DefaultDomain) +} + +// URL returns the public URL of this instance +func (wisski WissKI) URL() *url.URL { + // setup domain and path + url := &url.URL{ + Host: wisski.Domain(), + Path: "/", + } + + // use http or https scheme depending on if the distillery has it enabled + if wisski.instances.Config.HTTPSEnabled() { + url.Scheme = "https" + } else { + url.Scheme = "http" + } + + return url +} diff --git a/internal/component/instances/wisski_stack.go b/internal/component/instances/wisski_stack.go new file mode 100644 index 0000000..977bf7c --- /dev/null +++ b/internal/component/instances/wisski_stack.go @@ -0,0 +1,68 @@ +package instances + +import ( + "embed" + "io/fs" + "path/filepath" + + "github.com/FAU-CDI/wisski-distillery/internal/component" +) + +//go:embed all:instances/barrel instances/barrel.env +var barrelResources embed.FS + +// Barrel returns a stack representing the running WissKI Instance +func (wisski WissKI) Barrel() component.Installable { + return component.Installable{ + Stack: component.Stack{ + Dir: wisski.FilesystemBase, + }, + + Resources: barrelResources, + ContextPath: filepath.Join("instances", "barrel"), + EnvPath: filepath.Join("instances", "barrel.env"), + + EnvContext: map[string]string{ + "DATA_PATH": filepath.Join(wisski.FilesystemBase, "data"), + + "SLUG": wisski.Slug, + "VIRTUAL_HOST": wisski.Domain(), + + "LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()), + "LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail), + + "RUNTIME_DIR": wisski.instances.Config.RuntimeDir(), + "GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile, + }, + + MakeDirsPerm: fs.ModeDir | fs.ModePerm, + MakeDirs: []string{"data", ".composer"}, + + TouchFiles: []string{ + filepath.Join("data", "authorized_keys"), + }, + } +} + +//go:embed all:instances/reserve instances/reserve.env +var reserveResources embed.FS + +// Reserve returns a stack representing the reserve instance +func (wisski WissKI) Reserve() component.Installable { + return component.Installable{ + Stack: component.Stack{ + Dir: wisski.FilesystemBase, + }, + + Resources: reserveResources, + ContextPath: filepath.Join("instances", "reserve"), + EnvPath: filepath.Join("instances", "reserve.env"), + + EnvContext: map[string]string{ + "VIRTUAL_HOST": wisski.Domain(), + + "LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()), + "LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail), + }, + } +} diff --git a/internal/component/sql/database.go b/internal/component/sql/database.go index 31e6dea..b541139 100644 --- a/internal/component/sql/database.go +++ b/internal/component/sql/database.go @@ -63,7 +63,7 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code" // Backup makes a backup of the sql database into dest. func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error { - io = stream.NewIOStream(dest, io.Stderr, nil, 0) + io = io.Streams(dest, nil, nil, 0).NonInteractive() code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database) if err != nil { diff --git a/internal/wisski/backup.go b/internal/wisski/backup.go index 75d4ed2..f6e3c0e 100644 --- a/internal/wisski/backup.go +++ b/internal/wisski/backup.go @@ -211,8 +211,6 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { return } - iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0) - backup.InstanceSnapshots = make([]Snapshot, len(instances)) for i, instance := range instances { backup.InstanceSnapshots[i] = func() Snapshot { @@ -224,7 +222,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) { } files <- dir - return dis.Snapshot(instance, iochild, SnapshotDescription{ + return dis.Snapshot(instance, io.NonInteractive(), SnapshotDescription{ Dest: dir, }) }() diff --git a/internal/wisski/snapshot.go b/internal/wisski/snapshot.go index 9d992d6..34e3e7f 100644 --- a/internal/wisski/snapshot.go +++ b/internal/wisski/snapshot.go @@ -190,7 +190,7 @@ func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, d // makeBlackbox runs the blackbox backup of the system. // It pauses the Instance, if a consistent state is required. func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, instance instances.WissKI) { - stack := instance.Stack() + stack := instance.Barrel() og := opgroup.NewOpGroup[string](4)