rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Split up WissKI Instance code

Browse source
This commit is contained in:
Tom Wiesing 2022-09-15 12:54:16 +02:00
parent d7110dd63c
commit 85b5603d9d
No known key found for this signature in database
16 changed files with 285 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/provision.go
View file

@ -107,7 +107,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
// start the container!
logging.LogMessage(context.IOStream, "Starting Container")
if err := instance.Stack().Up(context.IOStream); err != nil {
if err := instance.Barrel().Up(context.IOStream); err != nil {
return err
}

2
cmd/purge.go
View file

@ -67,7 +67,7 @@ func (p purge) Run(context wisski_distillery.Context) error {
// remove docker stack
logging.LogMessage(context.IOStream, "Stopping and removing docker container")
if err := instance.Stack().Down(context.IOStream); err != nil {
if err := instance.Barrel().Down(context.IOStream); err != nil {
context.EPrintln(err)
}

2
cmd/rebuild.go
View file

@ -42,7 +42,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error {
var globalErr error
for _, instance := range instances {
logging.LogOperation(func() error {
s := instance.Stack()
s := instance.Barrel()
if err := logging.LogOperation(func() error {
return s.Install(context.IOStream, component.InstallationContext{})
}, context.IOStream, "Installing docker stack"); err != nil {

2
cmd/reserve.go
View file

@ -63,7 +63,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
}
// setup docker stack
s := instance.ReserveStack()
s := instance.Reserve()
{
if err := logging.LogOperation(func() error {
return s.Install(context.IOStream, component.InstallationContext{})

6
go.work Normal file
View file

@ -0,0 +1,6 @@
go 1.18
use (
.
../../tkw1536/goprogram
)

4
go.work.sum Normal file
View file

@ -0,0 +1,4 @@
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

56
internal/component/instances/wisski_create.go
View file

@ -3,8 +3,12 @@ package instances
import (
"errors"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
"github.com/alessio/shellescape"
"github.com/tkw1536/goprogram/stream"
)
var errInvalidSlug = errors.New("not a valid slug")
@ -59,3 +63,55 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
wisski.instances = instances
return wisski, nil
}
// Provision provisions an instance, assuming that the required databases already exist.
func (wisski WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Barrel()
if err := st.Install(io, component.InstallationContext{}); err != nil {
return err
}
// Pull and build the stack!
if err := st.Update(io, false); err != nil {
return err
}
provisionParams := []string{
wisski.Domain(),
wisski.SqlDatabase,
wisski.SqlUsername,
wisski.SqlPassword,
wisski.GraphDBRepository,
wisski.GraphDBUsername,
wisski.GraphDBPassword,
wisski.DrupalUsername,
wisski.DrupalPassword,
"", // TODO: DrupalVersion
"", // TODO: WissKIVersion
}
// escape the parameter
for i, param := range provisionParams {
provisionParams[i] = shellescape.Quote(param)
}
// figure out the provision script
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}
if code != 0 {
return errors.New("unable to run provision script")
}
return nil
}

235
internal/component/instances/wisski_db.go
View file

@ -1,25 +1,7 @@
package instances
import (
"bytes"
"embed"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/alessio/shellescape"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
// WissKI represents a single WissKI Instance
@ -66,220 +48,3 @@ func (wisski *WissKI) Delete() error {
// delete it directly
return db.Delete(&wisski.Instance).Error
}
// Shell executes a shell command inside the
func (wisski WissKI) Shell(io stream.IOStream, argv ...string) (int, error) {
return wisski.Stack().Exec(io, "barrel", "/bin/sh", append([]string{"/user_shell.sh"}, argv...)...)
}
// Domain returns the full domain name of this instance
func (wisski WissKI) Domain() string {
return fmt.Sprintf("%s.%s", wisski.Slug, wisski.instances.Config.DefaultDomain)
}
// URL returns the public URL of this instance
func (wisski WissKI) URL() *url.URL {
// setup domain and path
url := &url.URL{
Host: wisski.Domain(),
Path: "/",
}
// use http or https scheme depending on if the distillery has it enabled
if wisski.instances.Config.HTTPSEnabled() {
url.Scheme = "https"
} else {
url.Scheme = "http"
}
return url
}
//go:embed all:instances/barrel instances/barrel.env
var barrelResources embed.FS
// Stack represents a stack representing this instance
func (wisski WissKI) Stack() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},
Resources: barrelResources,
ContextPath: filepath.Join("instances", "barrel"),
EnvPath: filepath.Join("instances", "barrel.env"),
EnvContext: map[string]string{
"DATA_PATH": filepath.Join(wisski.FilesystemBase, "data"),
"SLUG": wisski.Slug,
"VIRTUAL_HOST": wisski.Domain(),
"LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()),
"LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail),
"RUNTIME_DIR": wisski.instances.Config.RuntimeDir(),
"GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
},
}
}
//go:embed all:instances/reserve instances/reserve.env
var reserveResources embed.FS
func (wisski WissKI) ReserveStack() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},
Resources: reserveResources,
ContextPath: filepath.Join("instances", "reserve"),
EnvPath: filepath.Join("instances", "reserve.env"),
EnvContext: map[string]string{
"VIRTUAL_HOST": wisski.Domain(),
"LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()),
"LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail),
},
}
}
// Provision provisions an instance, assuming that the required databases already exist.
func (wisski WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Stack()
if err := st.Install(io, component.InstallationContext{}); err != nil {
return err
}
// Pull and build the stack!
if err := st.Update(io, false); err != nil {
return err
}
provisionParams := []string{
wisski.Domain(),
wisski.SqlDatabase,
wisski.SqlUsername,
wisski.SqlPassword,
wisski.GraphDBRepository,
wisski.GraphDBUsername,
wisski.GraphDBPassword,
wisski.DrupalUsername,
wisski.DrupalPassword,
"", // TODO: DrupalVersion
"", // TODO: WissKIVersion
}
// escape the parameter
for i, param := range provisionParams {
provisionParams[i] = shellescape.Quote(param)
}
// figure out the provision script
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}
if code != 0 {
return errors.New("unable to run provision script")
}
return nil
}
// NoPrefix checks if this WissKI instance is excluded from generating prefixes
func (wisski *WissKI) NoPrefix() bool {
return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
// PrefixConfig returns the prefix config belonging to this instance.
func (wisski *WissKI) PrefixConfig() (config string, err error) {
// if the user requested to skip the prefix, then don't do anything with it!
if wisski.NoPrefix() {
return "", nil
}
var builder strings.Builder
// domain
builder.WriteString(wisski.URL().String() + ":")
builder.WriteString("\n")
// default prefixes
wu := stream.NewIOStream(&builder, nil, nil, 0)
code, err := wisski.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
if err != nil || code != 0 {
return "", errPrefixExecFailed
}
// custom prefixes
prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if err != nil {
return "", err
}
defer prefix.Close()
if _, err := io.Copy(&builder, prefix); err != nil {
return "", err
}
builder.WriteString("\n")
}
// and done!
return builder.String(), nil
}
var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder")
// ExportPathbuilders writes pathbuilders into the directory dest
func (wisski *WissKI) ExportPathbuilders(dest string) error {
// export all the pathbuilders into the buffer
var buffer bytes.Buffer
wu := stream.NewIOStream(&buffer, nil, nil, 0)
code, err := wisski.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php")
if err != nil || code != 0 {
return errPathbuildersExecFailed
}
// decode them as a json array
var pathbuilders map[string]string
if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil {
return err
}
// sort the names of the pathbuilders
names := maps.Keys(pathbuilders)
slices.Sort(names)
// write each into a file!
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
return err
}
}
return nil
}

8
internal/component/instances/wisski_exec.go Normal file
View file

@ -0,0 +1,8 @@
package instances
import "github.com/tkw1536/goprogram/stream"
// Shell executes a shell command inside the instance.
func (wisski WissKI) Shell(io stream.IOStream, argv ...string) (int, error) {
return wisski.Barrel().Exec(io, "barrel", "/bin/sh", append([]string{"/user_shell.sh"}, argv...)...)
}

49
internal/component/instances/wisski_pathbuilders.go Normal file
View file

@ -0,0 +1,49 @@
package instances
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder")
// ExportPathbuilders writes pathbuilders into the directory dest
func (wisski *WissKI) ExportPathbuilders(dest string) error {
// export all the pathbuilders into the buffer
var buffer bytes.Buffer
wu := stream.NewIOStream(&buffer, nil, nil, 0)
code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php")
if err != nil || code != 0 {
return errPathbuildersExecFailed
}
// decode them as a json array
var pathbuilders map[string]string
if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil {
return err
}
// sort the names of the pathbuilders
names := maps.Keys(pathbuilders)
slices.Sort(names)
// write each into a file!
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
return err
}
}
return nil
}

58
internal/component/instances/wisski_prefix.go Normal file
View file

@ -0,0 +1,58 @@
package instances
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/stream"
)
// NoPrefix checks if this WissKI instance is excluded from generating prefixes.
// TODO: Move this to the database!
func (wisski *WissKI) NoPrefix() bool {
return fsx.IsFile(filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
// PrefixConfig returns the prefix config belonging to this instance.
func (wisski *WissKI) PrefixConfig() (config string, err error) {
// if the user requested to skip the prefix, then don't do anything with it!
if wisski.NoPrefix() {
return "", nil
}
var builder strings.Builder
// domain
builder.WriteString(wisski.URL().String() + ":")
builder.WriteString("\n")
// default prefixes
wu := stream.NewIOStream(&builder, nil, nil, 0)
code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
if err != nil || code != 0 {
return "", errPrefixExecFailed
}
// custom prefixes
prefixPath := filepath.Join(wisski.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if err != nil {
return "", err
}
defer prefix.Close()
if _, err := io.Copy(&builder, prefix); err != nil {
return "", err
}
builder.WriteString("\n")
}
// and done!
return builder.String(), nil
}

29
internal/component/instances/wisski_props.go Normal file
View file

@ -0,0 +1,29 @@
package instances
import (
"fmt"
"net/url"
)
// Domain returns the full domain name of this WissKI
func (wisski WissKI) Domain() string {
return fmt.Sprintf("%s.%s", wisski.Slug, wisski.instances.Config.DefaultDomain)
}
// URL returns the public URL of this instance
func (wisski WissKI) URL() *url.URL {
// setup domain and path
url := &url.URL{
Host: wisski.Domain(),
Path: "/",
}
// use http or https scheme depending on if the distillery has it enabled
if wisski.instances.Config.HTTPSEnabled() {
url.Scheme = "https"
} else {
url.Scheme = "http"
}
return url
}

68
internal/component/instances/wisski_stack.go Normal file
View file

@ -0,0 +1,68 @@
package instances
import (
"embed"
"io/fs"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
//go:embed all:instances/barrel instances/barrel.env
var barrelResources embed.FS
// Barrel returns a stack representing the running WissKI Instance
func (wisski WissKI) Barrel() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},
Resources: barrelResources,
ContextPath: filepath.Join("instances", "barrel"),
EnvPath: filepath.Join("instances", "barrel.env"),
EnvContext: map[string]string{
"DATA_PATH": filepath.Join(wisski.FilesystemBase, "data"),
"SLUG": wisski.Slug,
"VIRTUAL_HOST": wisski.Domain(),
"LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()),
"LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail),
"RUNTIME_DIR": wisski.instances.Config.RuntimeDir(),
"GLOBAL_AUTHORIZED_KEYS_FILE": wisski.instances.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
},
}
}
//go:embed all:instances/reserve instances/reserve.env
var reserveResources embed.FS
// Reserve returns a stack representing the reserve instance
func (wisski WissKI) Reserve() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
},
Resources: reserveResources,
ContextPath: filepath.Join("instances", "reserve"),
EnvPath: filepath.Join("instances", "reserve.env"),
EnvContext: map[string]string{
"VIRTUAL_HOST": wisski.Domain(),
"LETSENCRYPT_HOST": wisski.instances.Config.IfHttps(wisski.Domain()),
"LETSENCRYPT_EMAIL": wisski.instances.Config.IfHttps(wisski.instances.Config.CertbotEmail),
},
}
}

2
internal/component/sql/database.go
View file

@ -63,7 +63,7 @@ var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code"
// Backup makes a backup of the sql database into dest.
func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {

4
internal/wisski/backup.go
View file

@ -211,8 +211,6 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
return
}
iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
backup.InstanceSnapshots = make([]Snapshot, len(instances))
for i, instance := range instances {
backup.InstanceSnapshots[i] = func() Snapshot {
@ -224,7 +222,7 @@ func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
}
files <- dir
return dis.Snapshot(instance, iochild, SnapshotDescription{
return dis.Snapshot(instance, io.NonInteractive(), SnapshotDescription{
Dest: dir,
})
}()

2
internal/wisski/snapshot.go
View file

@ -190,7 +190,7 @@ func (dis *Distillery) Snapshot(instance instances.WissKI, io stream.IOStream, d
// makeBlackbox runs the blackbox backup of the system.
// It pauses the Instance, if a consistent state is required.
func (snapshot *Snapshot) makeBlackbox(io stream.IOStream, dis *Distillery, instance instances.WissKI) {
stack := instance.Stack()
stack := instance.Barrel()
og := opgroup.NewOpGroup[string](4)