rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

system_update: Perform provisioning in parallel

Browse source
This commit is contained in:
Tom Wiesing 2022-09-22 18:06:46 +02:00
parent c091761762
commit 72d95f58ea
No known key found for this signature in database
15 changed files with 182 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

78
cmd/drupal_setting.go Normal file
View file

@ -0,0 +1,78 @@
package cmd
import (
"encoding/json"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/tkw1536/goprogram/exit"
)
// DrupalSetting is then 'drupal_setting' command
var DrupalSetting wisski_distillery.Command = setting{}
type setting struct {
Positionals struct {
Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to get or set value for"`
Setting string `positional-arg-name:"SETTING" require:"1-1" description:"name of setting to read or write"`
Value string `positional-arg-name:"VALUE" description:"json serialization of value to write"`
} `positional-args:"true"`
}
func (setting) Description() wisski_distillery.Description {
return wisski_distillery.Description{
Requirements: core.Requirements{
NeedsDistillery: true,
},
Command: "drupal_setting",
Description: "Get or set a drupal setting",
}
}
var errSettingGet = exit.Error{
ExitCode: exit.ExitGeneric,
Message: "Unable to get setting",
}
var errSettingSet = exit.Error{
ExitCode: exit.ExitGeneric,
Message: "Unable to set setting",
}
func (ds setting) Run(context wisski_distillery.Context) error {
instance, err := context.Environment.Instances().WissKI(ds.Positionals.Slug)
if err != nil {
return err
}
if ds.Positionals.Value == "" {
// get the setting
value, err := instance.GetSettingsPHP(ds.Positionals.Setting)
if err != nil {
return errSettingGet.Wrap(err)
}
// and print it
if err := json.NewEncoder(context.Stdout).Encode(value); err != nil {
return errSettingGet.Wrap(err)
}
// finish with a newline
context.Println("")
return nil
}
// serialize the setting into json
var data any
if err := json.Unmarshal([]byte(ds.Positionals.Value), &data); err != nil {
return errSettingSet.Wrap(err)
}
// set the serialized value!
if err := instance.SetSettingsPHP(ds.Positionals.Setting, data); err != nil {
return errSettingSet.Wrap(err)
}
// and we're done
return nil
}

46
cmd/system_update.go
View file

@ -1,6 +1,9 @@
package cmd
import (
"fmt"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/core"
@ -8,7 +11,9 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/lib/status"
"github.com/tkw1536/goprogram/parser"
"github.com/tkw1536/goprogram/stream"
)
// SystemUpdate is the 'system_update' command
@ -106,24 +111,35 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
}
if err := logging.LogOperation(func() error {
for _, component := range dis.Installables() {
name := component.Name()
stack := component.Stack(dis.Core.Environment)
ctx := component.Context(ctx)
group := &status.Group[component.Installable]{
Writer: context.Stdout,
PrefixString: func(item component.Installable, index int) string {
return fmt.Sprintf("[install %q]: ", item.Name())
},
PrefixAlign: true,
ErrString: func(item component.Installable, index int, err error) string {
if err == nil {
return "ok"
}
return "failed (" + err.Error() + ")"
},
Handler: func(item component.Installable, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
stack := item.Stack(context.Environment.Environment)
if err := logging.LogOperation(func() error {
return stack.Install(context.IOStream, ctx)
}, context.IOStream, "Installing Docker Stack %q", name); err != nil {
return err
}
if err := stack.Install(io, item.Context(ctx)); err != nil {
return err
}
if err := logging.LogOperation(func() error {
return stack.Update(context.IOStream, true)
}, context.IOStream, "Updating Docker Stack: %q", name); err != nil {
return err
}
if err := stack.Update(io, true); err != nil {
return err
}
return nil
},
}
return nil
return group.Run(dis.Installables())
}, context.IOStream, "Performing Stack Updates"); err != nil {
return err
}

2
cmd/wdcli/main.go
View file

@ -42,8 +42,10 @@ func init() {
wdcli.Register(cmd.Shell)
wdcli.Register(cmd.BlindUpdate)
wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration
wdcli.Register(cmd.Pathbuilders)
wdcli.Register(cmd.Prefixes)
wdcli.Register(cmd.DrupalSetting)
// backup & cron
wdcli.Register(cmd.Snapshot)

2
go.mod
View file

@ -9,7 +9,7 @@ require (
github.com/feiin/sqlstring v0.3.0
github.com/go-sql-driver/mysql v1.6.0
github.com/pkg/errors v0.9.1
github.com/tkw1536/goprogram v0.0.12
github.com/tkw1536/goprogram v0.0.13
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561
golang.org/x/sync v0.0.0-20220907140024-f12130a52804
gorm.io/driver/mysql v1.3.6

10
go.sum
View file

@ -19,16 +19,26 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/tkw1536/goprogram v0.0.12 h1:CJI79IieP750q9j27OmoB1J/PC4povjk8kdckPVJ1YQ=
github.com/tkw1536/goprogram v0.0.12/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4=
github.com/tkw1536/goprogram v0.0.13 h1:tq36MGZ24T+Xjv8y+bWbMGv2zx5prJ7tmi/bvs3wemA=
github.com/tkw1536/goprogram v0.0.13/go.mod h1:rX9MKOpJ9qAu4jHV2+n64SKmm3c2D3Hh1V8zC1H3jB4=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20220921164117-439092de6870 h1:j8b6j9gzSigH28O5SjSpQSSh9lFd6f5D/q0aHjNTulc=
golang.org/x/exp v0.0.0-20220921164117-439092de6870/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=
golang.org/x/sync v0.0.0-20220907140024-f12130a52804/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho=
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w=
golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM=
gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.9 h1:NSHG021i+MCznokeXR3udGaNyFyBQJW8MbjrJMVCfGw=
gorm.io/gorm v1.23.9/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=

3
internal/component/control/control.go
View file

@ -31,7 +31,7 @@ func (control Control) Path() string {
var resources embed.FS
func (control *Control) Stack(env environment.Environment) component.StackWithResources {
return component.MakeStack(control, env, component.StackWithResources{
stt := component.MakeStack(control, env, component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
@ -52,6 +52,7 @@ func (control *Control) Stack(env environment.Environment) component.StackWithRe
TouchFiles: []string{control.ResolverFile},
CopyContextFiles: []string{core.Executable},
})
return stt
}
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {

29
internal/component/instances/php/settings.php Normal file
View file

@ -0,0 +1,29 @@
<?php
/** gets a setting from 'settings.php' */
function get_setting($name) {
use \Drupal\Core\Site\Settings;
return Settings::get($name);
}
/** sets a setting in 'settings.php' */
function set_setting($name, $value) {
// load install.inc
if(is_file(DRUPAL_ROOT . "/internal/")) {
include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
} else {
include_once DRUPAL_ROOT . "/core/includes/install.inc";
}
// update the provided setting
$settings["settings"][$name] = (object)[
"value" => $value,
"required" => TRUE,
];
// find the filename
$filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
drupal_rewrite_settings($settings, $filename);
return True;
}

6
internal/component/instances/wisski_pathbuilders.go
View file

@ -17,7 +17,7 @@ var exportPathbuilderPHP string
// Pathbuilders returns the ids of all pathbuilders in consistent order.
func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &ids, exportPathbuilderPHP, "all_list")
err = wisski.ExecPHPScript(stream.FromDebug(), &ids, exportPathbuilderPHP, "all_list")
slices.Sort(ids)
return
}
@ -25,13 +25,13 @@ func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
// Pathbuilder returns a single pathbuilder as xml.
// If it does not exist, it returns the empty string and nil error.
func (wisski *WissKI) Pathbuilder(id string) (xml string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &xml, exportPathbuilderPHP, "one_xml", id)
err = wisski.ExecPHPScript(stream.FromDebug(), &xml, exportPathbuilderPHP, "one_xml", id)
return
}
// AllPathbuilders returns all pathbuilders serialized as xml
func (wisski *WissKI) AllPathbuilders() (pathbuilders map[string]string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &pathbuilders, exportPathbuilderPHP, "all_xml")
err = wisski.ExecPHPScript(stream.FromDebug(), &pathbuilders, exportPathbuilderPHP, "all_xml")
return
}

16
internal/component/instances/wisski_php.go
View file

@ -7,6 +7,8 @@ import (
"strings"
"github.com/tkw1536/goprogram/stream"
_ "embed"
)
var ErrExecInvalidCode = errors.New("invalid code to execute")
@ -123,3 +125,17 @@ func marshalPHP(data any) (string, error) {
result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
return result, nil
}
//
//go:embed php/settings.php
var settingsPHP string
func (wisski *WissKI) GetSettingsPHP(key string) (value any, err error) {
err = wisski.ExecPHPScript(stream.FromDebug(), &value, settingsPHP, "get_setting", key)
return
}
func (wisski *WissKI) SetSettingsPHP(key string, value any) error {
return wisski.ExecPHPScript(stream.FromDebug(), nil, settingsPHP, "set_setting", key, value)
}

2
internal/component/instances/wisski_prefix.go
View file

@ -25,7 +25,7 @@ var listURIPrefixesPHP string
// Prefixes returns the prefixes
func (wisski *WissKI) Prefixes() (prefixes []string, err error) {
// get all the ugly prefixes
err = wisski.ExecPHPScript(stream.FromNil(), &prefixes, listURIPrefixesPHP, "list_prefixes")
err = wisski.ExecPHPScript(stream.FromDebug(), &prefixes, listURIPrefixesPHP, "list_prefixes")
if err != nil {
return nil, err
}

4
internal/component/sql/connect.go
View file

@ -8,6 +8,7 @@ import (
"sync/atomic"
mysqldriver "github.com/go-sql-driver/mysql"
"github.com/tkw1536/goprogram/stream"
"gorm.io/driver/mysql"
"gorm.io/gorm"
@ -43,7 +44,6 @@ func (sql *SQL) Exec(query string, args ...interface{}) error {
func (sql *SQL) WaitExec() error {
return wait.Wait(func() bool {
err := sql.Exec("select 1;")
// log.Printf("[WaitQuery] %s\n", err) // debug
return err == nil
}, sql.PollInterval, sql.PollContext)
}
@ -90,8 +90,10 @@ func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
// WaitQueryTable waits for a connection to succeed via QueryTable
func (sql *SQL) WaitQueryTable() error {
n := stream.FromDebug()
return wait.Wait(func() bool {
_, err := sql.QueryTable(true, models.InstanceTable)
n.EPrintf("[SQL.WaitQueryTable]: %s\n", err)
return err == nil
}, sql.PollInterval, sql.PollContext)
}

4
internal/component/sql/update.go
View file

@ -21,10 +21,10 @@ func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) {
// unsafeWaitShell waits for a connection via the database shell to succeed
func (sql *SQL) unsafeWaitShell() error {
n := stream.FromNil()
n := stream.FromDebug()
return wait.Wait(func() bool {
code, err := sql.Shell(n, "-e", "select 1;")
// log.Printf("[unsafeWaitShell] %d %s\n", code, err) // debug
n.EPrintf("[SQL.unsafeWaitShell]: %d %s\n", code, err)
return err == nil && code == 0
}, sql.PollInterval, sql.PollContext)
}

2
internal/component/stack.go
View file

@ -89,7 +89,7 @@ func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...str
return ds.compose(io, compose...)
}
// Run executes the provided service with the given executable.
// Run runs a command in a running container with the given executable.
// It is equivalent to 'docker compose run [--rm] $service $executable $args...'.
//
// It returns the exit code of the process.

3
internal/component/triplestore/database.go
View file

@ -9,6 +9,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
type TriplestoreUserPayload struct {
@ -85,8 +86,10 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
// Wait waits for the connection to the Triplestore to succeed.
// This is achieved using a polling strategy.
func (ts Triplestore) Wait() error {
n := stream.FromDebug()
return wait.Wait(func() bool {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
n.EPrintf("[Triplestore.Wait]: %s\n", err)
if err != nil {
return false
}

2
internal/component/web/web.go
View file

@ -2,7 +2,6 @@ package web
import (
"embed"
"fmt"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/component"
@ -22,7 +21,6 @@ func (Web) Name() string {
func (web Web) Path() string {
res := filepath.Join(web.Core.Config.DeployRoot, "core", web.Name())
fmt.Println("debug====" + res)
return res
}