diff --git a/cmd/purge.go b/cmd/purge.go
index 39f6faf..7183fbf 100644
--- a/cmd/purge.go
+++ b/cmd/purge.go
@@ -114,6 +114,11 @@ func (p purge) Run(context wisski_distillery.Context) error {
context.EPrintln(err)
}
+ logging.LogMessage(context.IOStream, "Purging instance metadata")
+ if err := instance.Metadata().Purge(); err != nil {
+ context.EPrintln(err)
+ }
+
logging.LogMessage(context.IOStream, "Instance %s has been purged", slug)
return nil
}
diff --git a/cmd/rebuild.go b/cmd/rebuild.go
index 8dbbfba..2e86974 100644
--- a/cmd/rebuild.go
+++ b/cmd/rebuild.go
@@ -2,7 +2,6 @@ package cmd
import (
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
- "github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/exit"
@@ -44,22 +43,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error {
var globalErr error
for _, instance := range instances {
logging.LogOperation(func() error {
- s := instance.Barrel()
- if err := logging.LogOperation(func() error {
- return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
- }, context.IOStream, "Installing docker stack"); err != nil {
- globalErr = err
- return err
- }
-
- if err := logging.LogOperation(func() error {
- return s.Update(context.IOStream, true)
- }, context.IOStream, "Updating docker stack"); err != nil {
- globalErr = err
- return err
- }
-
- return nil
+ return instance.Build(context.IOStream, true)
}, context.IOStream, "Rebuilding instance %s", instance.Slug)
}
diff --git a/cmd/reserve.go b/cmd/reserve.go
index 40fd05d..4477b64 100644
--- a/cmd/reserve.go
+++ b/cmd/reserve.go
@@ -66,7 +66,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
s := instance.Reserve()
{
if err := logging.LogOperation(func() error {
- return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
+ return s.Install(context.IOStream, component.InstallationContext{})
}, context.IOStream, "Installing docker stack"); err != nil {
return err
}
diff --git a/cmd/system_update.go b/cmd/system_update.go
index 84c837e..a9801b4 100644
--- a/cmd/system_update.go
+++ b/cmd/system_update.go
@@ -112,7 +112,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
ctx := component.Context(ctx)
if err := logging.LogOperation(func() error {
- return stack.Install(dis.Core.Environment, context.IOStream, ctx)
+ return stack.Install(context.IOStream, ctx)
}, context.IOStream, "Installing Docker Stack %q", name); err != nil {
return err
}
diff --git a/internal/component/control/html/instance.html b/internal/component/control/html/instance.html
index e8c5d42..e5cbae1 100644
--- a/internal/component/control/html/instance.html
+++ b/internal/component/control/html/instance.html
@@ -13,6 +13,7 @@
URL: {{ .Info.URL }}
Running: {{ .Info.Running }}
+ Last Rebuild: {{ .Info.LastRebuild }}
Created: {{ .Instance.Created }}
OwnerEmail: {{ .Instance.OwnerEmail }}
diff --git a/internal/component/instances/meta.go b/internal/component/instances/meta.go
new file mode 100644
index 0000000..d1aa9e3
--- /dev/null
+++ b/internal/component/instances/meta.go
@@ -0,0 +1,204 @@
+package instances
+
+import (
+ "encoding/json"
+ "errors"
+
+ "github.com/FAU-CDI/wisski-distillery/internal/component/sql"
+ "github.com/FAU-CDI/wisski-distillery/internal/models"
+ "gorm.io/gorm"
+)
+
+// MetaKey represents a key for metadata.
+type MetaKey string
+
+// ErrMetadatumNotSet is returned by various [MetaStorage] functions when a metadatum is not set
+var ErrMetadatumNotSet = errors.New("metadatum not set")
+
+// MetaStorage manages some metadata.
+type MetaStorage interface {
+ // Get retrieves metadata with the provided key and deserializes the first one into target.
+ // If no metadatum exists, returns [ErrMetadatumNotSet].
+ Get(key MetaKey, target any) error
+
+ // GetAll receives all metadata with the provided keys.
+ // For each received value, the targets function is called with the current index, and total number of results.
+ // The function is intended to return a target for deserialization.
+ //
+ // When no metadatum exists, targets is not called, and nil error is returned.
+ GetAll(key MetaKey, targets func(index, total int) any) error
+
+ // Delete deletes all metadata with the provided key.
+ Delete(key MetaKey) error
+
+ // Set serializes value and stores it with the provided key.
+ // Any other metadata with the same key is deleted.
+ Set(key MetaKey, value any) error
+
+ // Add serializes values and stores each as associated with the provided key.
+ // Already existing metadata is left intact.
+ Add(key MetaKey, values ...any) error
+
+ // Purge removes all metadata, regardless of key.
+ Purge() error
+}
+
+// Metadata returns a system-wide [MetaStorage].
+func (instances *Instances) Metadata() MetaStorage {
+ return &storage{
+ SQL: instances.SQL,
+ Slug: "", // not associated to any slug
+ }
+}
+
+// Metadata returns a [MetaStorage] that manages metadata related to this WissKI instance.
+// It will be automatically deleted once the instance is deleted.
+func (wisski *WissKI) Metadata() MetaStorage {
+ return &storage{
+ SQL: wisski.instances.SQL,
+ Slug: wisski.Slug, // associated to this instance
+ }
+}
+
+// storage implements MetaStorage
+type storage struct {
+ SQL *sql.SQL
+ Slug string
+}
+
+func (s *storage) Get(key MetaKey, target any) error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ // read the datum from the database
+ var datum models.Metadatum
+ status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Order("pk DESC").Find(&datum)
+
+ // check if there was an error
+ if err := status.Error; err != nil {
+ return err
+ }
+
+ // check if e actually found it!
+ if status.RowsAffected == 0 {
+ return ErrMetadatumNotSet
+ }
+
+ // and do the unmarshaling!
+ return json.Unmarshal(datum.Value, target)
+}
+
+func (s *storage) GetAll(key MetaKey, target func(index, total int) any) error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ // read the datum from the database
+ var data []models.Metadatum
+ status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Find(&data)
+
+ // check if there was an error
+ if err := status.Error; err != nil {
+ return err
+ }
+
+ // unpack all of them into the destination
+ for index, datum := range data {
+ err := json.Unmarshal(datum.Value, target(index, len(data)))
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (s *storage) Delete(key MetaKey) error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ // delete all the values
+ status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
+ if err := status.Error; err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *storage) Set(key MetaKey, value any) error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ // marshal the value
+ bytes, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+
+ return table.Transaction(func(tx *gorm.DB) error {
+ // delete the old values
+ status := tx.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
+ if err := status.Error; err != nil {
+ return err
+ }
+
+ // create the new item to insert
+ status = tx.Create(&models.Metadatum{
+ Key: string(key),
+ Slug: s.Slug,
+ Value: bytes,
+ })
+ if err := status.Error; err != nil {
+ return err
+ }
+
+ return nil
+ })
+}
+
+func (s *storage) Add(key MetaKey, values ...any) error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ return table.Transaction(func(tx *gorm.DB) error {
+ for _, value := range values {
+ bytes, err := json.Marshal(value)
+ if err != nil {
+ return err
+ }
+
+ // create the new item to insert
+ status := tx.Create(&models.Metadatum{
+ Key: string(key),
+ Slug: s.Slug,
+ Value: bytes,
+ })
+ if err := status.Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
+
+func (s *storage) Purge() error {
+ table, err := s.SQL.QueryTable(true, models.MetadataTable)
+ if err != nil {
+ return err
+ }
+
+ status := table.Where("slug = ?", s.Slug).Delete(&models.Metadatum{})
+ if status.Error != nil {
+ return status.Error
+ }
+ return nil
+}
diff --git a/internal/component/instances/wisski_create.go b/internal/component/instances/wisski_create.go
index e8f8ff7..af1c75d 100644
--- a/internal/component/instances/wisski_create.go
+++ b/internal/component/instances/wisski_create.go
@@ -5,7 +5,6 @@ import (
"path/filepath"
"strings"
- "github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
"github.com/alessio/shellescape"
"github.com/tkw1536/goprogram/stream"
@@ -66,16 +65,10 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
}
// Provision provisions an instance, assuming that the required databases already exist.
-func (wisski WissKI) Provision(io stream.IOStream) error {
+func (wisski *WissKI) Provision(io stream.IOStream) error {
- // create the basic st!
- st := wisski.Barrel()
- if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
- return err
- }
-
- // Pull and build the stack!
- if err := st.Update(io, false); err != nil {
+ // build the container
+ if err := wisski.Build(io, false); err != nil {
return err
}
@@ -106,7 +99,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
- code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
+ code, err := wisski.Barrel().Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}
diff --git a/internal/component/instances/wisski_stack.go b/internal/component/instances/wisski_stack.go
index 461c68b..48f7bbf 100644
--- a/internal/component/instances/wisski_stack.go
+++ b/internal/component/instances/wisski_stack.go
@@ -3,15 +3,17 @@ package instances
import (
"embed"
"path/filepath"
+ "time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
+ "github.com/tkw1536/goprogram/stream"
)
//go:embed all:instances/barrel instances/barrel.env
var barrelResources embed.FS
// Barrel returns a stack representing the running WissKI Instance
-func (wisski WissKI) Barrel() component.StackWithResources {
+func (wisski *WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
@@ -43,11 +45,59 @@ func (wisski WissKI) Barrel() component.StackWithResources {
}
}
+const KeyLastRebuild MetaKey = "lastRebuild"
+
+func (wisski *WissKI) LastRebuild() (t time.Time, err error) {
+ var epoch int64
+
+ // read the epoch!
+ err = wisski.Metadata().Get(KeyLastRebuild, &epoch)
+ if err == ErrMetadatumNotSet {
+ return t, nil
+ }
+ if err != nil {
+ return t, err
+ }
+
+ // and turn it into time!
+ return time.Unix(epoch, 0), nil
+}
+
+func (wisski *WissKI) setLastRebuild() error {
+ return wisski.Metadata().Set(KeyLastRebuild, time.Now().Unix())
+}
+
+// Build builds or rebuilds the barel connected to this instance.
+//
+// It also logs the current time into the metadata belonging to this instance.
+func (wisski *WissKI) Build(stream stream.IOStream, start bool) error {
+ barrel := wisski.Barrel()
+
+ var context component.InstallationContext
+
+ {
+ err := barrel.Install(stream, context)
+ if err != nil {
+ return err
+ }
+ }
+
+ {
+ err := barrel.Update(stream, start)
+ if err != nil {
+ return err
+ }
+ }
+
+ // store the current last rebuild
+ return wisski.setLastRebuild()
+}
+
//go:embed all:instances/reserve instances/reserve.env
var reserveResources embed.FS
// Reserve returns a stack representing the reserve instance
-func (wisski WissKI) Reserve() component.StackWithResources {
+func (wisski *WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
diff --git a/internal/component/instances/wisski_status.go b/internal/component/instances/wisski_status.go
index 46eea7e..00827d8 100644
--- a/internal/component/instances/wisski_status.go
+++ b/internal/component/instances/wisski_status.go
@@ -1,6 +1,9 @@
package instances
import (
+ "fmt"
+ "time"
+
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
)
@@ -10,12 +13,15 @@ type Info struct {
Slug string // The slug of the instance
URL string // The public URL of this instance
+ LastRebuild time.Time
+
Running bool // is the instance running?
Pathbuilders []string // list of pathbuilders
}
// Info returns information about this WissKI instance.
func (wisski *WissKI) Info(quick bool) (info Info, err error) {
+ fmt.Println("call to info")
// static properties
info.Slug = wisski.Slug
info.URL = wisski.URL().String()
@@ -30,15 +36,20 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) {
})
// slower checks for extra properties.
- // these execute php code
+ // these might execute php code or require additional database queries.
if !quick {
+ group.Go(func() error {
+ info.Pathbuilders, _ = wisski.Pathbuilders()
+ return nil
+ })
group.Go(func() (err error) {
- info.Pathbuilders, err = wisski.Pathbuilders()
- return
+ info.LastRebuild, _ = wisski.LastRebuild()
+ return nil
})
}
err = group.Wait()
+ fmt.Println(err)
return
}
diff --git a/internal/component/sql/connect.go b/internal/component/sql/connect.go
index 35afe33..9c83136 100644
--- a/internal/component/sql/connect.go
+++ b/internal/component/sql/connect.go
@@ -21,8 +21,8 @@ import (
// ========== low-level connection ==========
//
-// Query performs a database query, outside a database contect
-func (sql *SQL) Query(query string, args ...interface{}) error {
+// Exec executes a database-independent database query.
+func (sql *SQL) Exec(query string, args ...interface{}) error {
// connect to the server
conn, err := sql.connect("")
if err != nil {
@@ -39,10 +39,10 @@ func (sql *SQL) Query(query string, args ...interface{}) error {
}
}
-// WaitQuery waits for the query interface to be able to connect to the database
-func (sql *SQL) WaitQuery() error {
+// WaitExec waits for the query interface to be able to connect to the database
+func (sql *SQL) WaitExec() error {
return wait.Wait(func() bool {
- err := sql.Query("select 1;")
+ err := sql.Exec("select 1;")
// log.Printf("[WaitQuery] %s\n", err) // debug
return err == nil
}, sql.PollInterval, sql.PollContext)
@@ -52,8 +52,8 @@ func (sql *SQL) WaitQuery() error {
// ========== connection via gorm ==========
//
-// QueryTable returns a gorm.DB to connect to the provided gorm database table
-func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
+// QueryTable returns a gorm.DB to connect to the provided distillery database table
+func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
conn, err := sql.connect(sql.Config.DistilleryDatabase)
if err != nil {
return nil, err
@@ -79,7 +79,7 @@ func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
}
// set the table
- db = db.Table(name)
+ db = db.Table(table)
// check that nothing went wrong
if db.Error != nil {
diff --git a/internal/component/sql/connect_shell.go b/internal/component/sql/connect_shell.go
deleted file mode 100644
index e4b317b..0000000
--- a/internal/component/sql/connect_shell.go
+++ /dev/null
@@ -1 +0,0 @@
-package sql
diff --git a/internal/component/sql/provision.go b/internal/component/sql/provision.go
index 6f147d6..912d21b 100644
--- a/internal/component/sql/provision.go
+++ b/internal/component/sql/provision.go
@@ -9,7 +9,10 @@ import (
var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters")
var errProvisionInvalidGrant = errors.New("Provision: Grant failed")
-// Provision provisions a new sql database and user
+// Provision creates a new database with the given name.
+// It then generates a new user, with the name 'user' and the password 'password', that is then granted access to this database.
+//
+// Provision internally waits for the database to become available.
func (sql *SQL) Provision(name, user, password string) error {
// NOTE(twiesing): We shouldn't use string concat to build sql queries.
@@ -41,6 +44,10 @@ func (sql *SQL) Provision(name, user, password string) error {
var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed")
+// CreateSuperuser createsa new user, with the name 'user' and the password 'password'.
+// It then grants this user superuser status in the database.
+//
+// CreateSuperuser internally waits for the database to become available.
func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error {
// NOTE(twiesing): This function unsafely uses the shell directly to create a superuser.
// This is for two reasons:
@@ -70,9 +77,21 @@ func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error
return nil
}
+var errPurgeUser = errors.New("PurgeUser: Failed to drop user")
+
// SQLPurgeUser deletes the specified user from the database
func (sql *SQL) PurgeUser(user string) error {
- return sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user)
+ if !sqle.IsSafeDatabaseSingleQuote(user) {
+ return errPurgeUser
+ }
+
+ query := "DROP USER IF EXISTS '" + user + "'@'%';" +
+ "FLUSH PRIVILEGES;"
+ if !sql.unsafeQueryShell(query) {
+ return errPurgeUser
+ }
+
+ return nil
}
var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name")
@@ -82,5 +101,5 @@ func (sql *SQL) PurgeDatabase(db string) error {
if !sqle.IsSafeDatabaseLiteral(db) {
return errSQLPurgeDB
}
- return sql.Query("DROP DATABASE IF EXISTS `" + db + "`")
+ return sql.Exec("DROP DATABASE IF EXISTS `" + db + "`")
}
diff --git a/internal/component/sql/update.go b/internal/component/sql/update.go
index 9372daf..0cdbc7c 100644
--- a/internal/component/sql/update.go
+++ b/internal/component/sql/update.go
@@ -8,6 +8,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
+ "github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/stream"
)
@@ -36,6 +37,10 @@ func (sql *SQL) unsafeQueryShell(query string) bool {
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name")
+var errSQLUnableToMigrate = exit.Error{
+ Message: "unable to migrate %s table: %s",
+ ExitCode: exit.ExitGeneric,
+}
// Update initializes or updates the SQL database.
func (sql *SQL) Update(io stream.IOStream) error {
@@ -62,7 +67,7 @@ func (sql *SQL) Update(io stream.IOStream) error {
return errSQLUnsafeDatabaseName
}
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase)
- if err := sql.Query(createDBSQL); err != nil {
+ if err := sql.Exec(createDBSQL); err != nil {
return err
}
}
@@ -71,18 +76,36 @@ func (sql *SQL) Update(io stream.IOStream) error {
logging.LogMessage(io, "Waiting for database update to be complete")
sql.WaitQueryTable()
- // open the database
- logging.LogMessage(io, "Migrating instances table")
- {
- db, err := sql.QueryTable(false, models.InstanceTable)
- if err != nil {
- return fmt.Errorf("unable to access bookkeeping table: %s", err)
- }
-
- if err := db.AutoMigrate(&models.Instance{}); err != nil {
- return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
- }
+ tables := []struct {
+ name string
+ model any
+ table string
+ }{
+ {
+ "instance",
+ &models.Instance{},
+ models.InstanceTable,
+ },
+ {
+ "metadata",
+ &models.Metadatum{},
+ models.MetadataTable,
+ },
}
- return nil
+ // migrate all of the tables!
+ return logging.LogOperation(func() error {
+ for _, table := range tables {
+ logging.LogMessage(io, "migrating %q table", table.name)
+ db, err := sql.QueryTable(false, table.table)
+ if err != nil {
+ return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
+ }
+
+ if err := db.AutoMigrate(table.model); err != nil {
+ return errSQLUnableToMigrate.WithMessageF(table.name, err)
+ }
+ }
+ return nil
+ }, io, "migrating database tables")
}
diff --git a/internal/component/stack.go b/internal/component/stack.go
index 809741b..8e0c1d5 100644
--- a/internal/component/stack.go
+++ b/internal/component/stack.go
@@ -217,7 +217,8 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
-func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
+func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
+ env := is.Stack.Env
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(
diff --git a/internal/models/metadata.go b/internal/models/metadata.go
new file mode 100644
index 0000000..955bfcc
--- /dev/null
+++ b/internal/models/metadata.go
@@ -0,0 +1,13 @@
+package models
+
+// MetadataTable is the name of the table the 'Metadatum' model is stored in.
+const MetadataTable = "metadatum"
+
+// Metadatum represents
+type Metadatum struct {
+ Pk uint `gorm:"column:pk;primaryKey"`
+
+ Key string `gorm:"column:key;not null"` // key for the value, see the keys below
+ Slug string `gorm:"column:slug"` // optional slug of instance
+ Value []byte `gorm:"column:value"` // serialized json value of the data
+}
diff --git a/internal/wisski/snapshot.go b/internal/wisski/snapshot.go
index dd13767..11bb39a 100644
--- a/internal/wisski/snapshot.go
+++ b/internal/wisski/snapshot.go
@@ -76,6 +76,7 @@ func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err er
// SnapshotDescription is a description for a snapshot
type SnapshotDescription struct {
Dest string // destination path
+ Log bool // should we log the creation of this snapshot?
Keepalive bool // should we keep the instance alive while making the snapshot?
}