Add a metadata system

cmd/purge.go

@@ -114,6 +114,11 @@ func (p purge) Run(context wisski_distillery.Context) error {
 		context.EPrintln(err)
 	}
 
+	logging.LogMessage(context.IOStream, "Purging instance metadata")
+	if err := instance.Metadata().Purge(); err != nil {
+		context.EPrintln(err)
+	}
+
 	logging.LogMessage(context.IOStream, "Instance %s has been purged", slug)
 	return nil
 }

cmd/rebuild.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
-	"github.com/FAU-CDI/wisski-distillery/internal/component"
 	"github.com/FAU-CDI/wisski-distillery/internal/core"
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/tkw1536/goprogram/exit"
@@ -44,22 +43,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error {
 	var globalErr error
 	for _, instance := range instances {
 		logging.LogOperation(func() error {
-			s := instance.Barrel()
-			if err := logging.LogOperation(func() error {
-				return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
-			}, context.IOStream, "Installing docker stack"); err != nil {
-				globalErr = err
-				return err
-			}
-
-			if err := logging.LogOperation(func() error {
-				return s.Update(context.IOStream, true)
-			}, context.IOStream, "Updating docker stack"); err != nil {
-				globalErr = err
-				return err
-			}
-
-			return nil
+			return instance.Build(context.IOStream, true)
 		}, context.IOStream, "Rebuilding instance %s", instance.Slug)
 	}
 

cmd/reserve.go

@@ -66,7 +66,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
 	s := instance.Reserve()
 	{
 		if err := logging.LogOperation(func() error {
-			return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{})
+			return s.Install(context.IOStream, component.InstallationContext{})
 		}, context.IOStream, "Installing docker stack"); err != nil {
 			return err
 		}

cmd/system_update.go

@@ -112,7 +112,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
 			ctx := component.Context(ctx)
 
 			if err := logging.LogOperation(func() error {
-				return stack.Install(dis.Core.Environment, context.IOStream, ctx)
+				return stack.Install(context.IOStream, ctx)
 			}, context.IOStream, "Installing Docker Stack %q", name); err != nil {
 				return err
 			}

internal/component/control/html/instance.html

@@ -13,6 +13,7 @@
     <b>URL:</b> <a href="{{ .Info.URL }}" target="_blank" rel="noopener noreferrer">{{ .Info.URL }}</a> <br />
     <hr />
     <b>Running:</b> <code>{{ .Info.Running }}</code> <br />
+    <b>Last Rebuild:</b> <code>{{ .Info.LastRebuild }}</code> <br />
     <hr />
     <b>Created:</b> <code>{{ .Instance.Created }}</code> <br />
     <b>OwnerEmail:</b> <code>{{ .Instance.OwnerEmail }}</code> <br />

internal/component/instances/meta.go

@@ -0,0 +1,204 @@
+package instances
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"gorm.io/gorm"
+)
+
+// MetaKey represents a key for metadata.
+type MetaKey string
+
+// ErrMetadatumNotSet is returned by various [MetaStorage] functions when a metadatum is not set
+var ErrMetadatumNotSet = errors.New("metadatum not set")
+
+// MetaStorage manages some metadata.
+type MetaStorage interface {
+	// Get retrieves metadata with the provided key and deserializes the first one into target.
+	// If no metadatum exists, returns [ErrMetadatumNotSet].
+	Get(key MetaKey, target any) error
+
+	// GetAll receives all metadata with the provided keys.
+	// For each received value, the targets function is called with the current index, and total number of results.
+	// The function is intended to return a target for deserialization.
+	//
+	// When no metadatum exists, targets is not called, and nil error is returned.
+	GetAll(key MetaKey, targets func(index, total int) any) error
+
+	// Delete deletes all metadata with the provided key.
+	Delete(key MetaKey) error
+
+	// Set serializes value and stores it with the provided key.
+	// Any other metadata with the same key is deleted.
+	Set(key MetaKey, value any) error
+
+	// Add serializes values and stores each as associated with the provided key.
+	// Already existing metadata is left intact.
+	Add(key MetaKey, values ...any) error
+
+	// Purge removes all metadata, regardless of key.
+	Purge() error
+}
+
+// Metadata returns a system-wide [MetaStorage].
+func (instances *Instances) Metadata() MetaStorage {
+	return &storage{
+		SQL:  instances.SQL,
+		Slug: "", // not associated to any slug
+	}
+}
+
+// Metadata returns a [MetaStorage] that manages metadata related to this WissKI instance.
+// It will be automatically deleted once the instance is deleted.
+func (wisski *WissKI) Metadata() MetaStorage {
+	return &storage{
+		SQL:  wisski.instances.SQL,
+		Slug: wisski.Slug, // associated to this instance
+	}
+}
+
+// storage implements MetaStorage
+type storage struct {
+	SQL  *sql.SQL
+	Slug string
+}
+
+func (s *storage) Get(key MetaKey, target any) error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	// read the datum from the database
+	var datum models.Metadatum
+	status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Order("pk DESC").Find(&datum)
+
+	// check if there was an error
+	if err := status.Error; err != nil {
+		return err
+	}
+
+	// check if e actually found it!
+	if status.RowsAffected == 0 {
+		return ErrMetadatumNotSet
+	}
+
+	// and do the unmarshaling!
+	return json.Unmarshal(datum.Value, target)
+}
+
+func (s *storage) GetAll(key MetaKey, target func(index, total int) any) error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	// read the datum from the database
+	var data []models.Metadatum
+	status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Find(&data)
+
+	// check if there was an error
+	if err := status.Error; err != nil {
+		return err
+	}
+
+	// unpack all of them into the destination
+	for index, datum := range data {
+		err := json.Unmarshal(datum.Value, target(index, len(data)))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *storage) Delete(key MetaKey) error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	// delete all the values
+	status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
+	if err := status.Error; err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *storage) Set(key MetaKey, value any) error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	// marshal the value
+	bytes, err := json.Marshal(value)
+	if err != nil {
+		return err
+	}
+
+	return table.Transaction(func(tx *gorm.DB) error {
+		// delete the old values
+		status := tx.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
+		if err := status.Error; err != nil {
+			return err
+		}
+
+		// create the new item to insert
+		status = tx.Create(&models.Metadatum{
+			Key:   string(key),
+			Slug:  s.Slug,
+			Value: bytes,
+		})
+		if err := status.Error; err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func (s *storage) Add(key MetaKey, values ...any) error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	return table.Transaction(func(tx *gorm.DB) error {
+		for _, value := range values {
+			bytes, err := json.Marshal(value)
+			if err != nil {
+				return err
+			}
+
+			// create the new item to insert
+			status := tx.Create(&models.Metadatum{
+				Key:   string(key),
+				Slug:  s.Slug,
+				Value: bytes,
+			})
+			if err := status.Error; err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func (s *storage) Purge() error {
+	table, err := s.SQL.QueryTable(true, models.MetadataTable)
+	if err != nil {
+		return err
+	}
+
+	status := table.Where("slug = ?", s.Slug).Delete(&models.Metadatum{})
+	if status.Error != nil {
+		return status.Error
+	}
+	return nil
+}

internal/component/instances/wisski_create.go

@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/FAU-CDI/wisski-distillery/internal/component"
 	"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
 	"github.com/alessio/shellescape"
 	"github.com/tkw1536/goprogram/stream"
@@ -66,16 +65,10 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
 }
 
 // Provision provisions an instance, assuming that the required databases already exist.
-func (wisski WissKI) Provision(io stream.IOStream) error {
+func (wisski *WissKI) Provision(io stream.IOStream) error {
 
-	// create the basic st!
-	st := wisski.Barrel()
-	if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
-		return err
-	}
-
-	// Pull and build the stack!
-	if err := st.Update(io, false); err != nil {
+	// build the container
+	if err := wisski.Build(io, false); err != nil {
 		return err
 	}
 
@@ -106,7 +99,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
 	// TODO: Move the provision script into the control plane!
 	provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
 
-	code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
+	code, err := wisski.Barrel().Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
 	if err != nil {
 		return err
 	}

internal/component/instances/wisski_stack.go

@@ -3,15 +3,17 @@ package instances
 import (
 	"embed"
 	"path/filepath"
+	"time"
 
 	"github.com/FAU-CDI/wisski-distillery/internal/component"
+	"github.com/tkw1536/goprogram/stream"
 )
 
 //go:embed all:instances/barrel instances/barrel.env
 var barrelResources embed.FS
 
 // Barrel returns a stack representing the running WissKI Instance
-func (wisski WissKI) Barrel() component.StackWithResources {
+func (wisski *WissKI) Barrel() component.StackWithResources {
 	return component.StackWithResources{
 		Stack: component.Stack{
 			Dir: wisski.FilesystemBase,
@@ -43,11 +45,59 @@ func (wisski WissKI) Barrel() component.StackWithResources {
 	}
 }
 
+const KeyLastRebuild MetaKey = "lastRebuild"
+
+func (wisski *WissKI) LastRebuild() (t time.Time, err error) {
+	var epoch int64
+
+	// read the epoch!
+	err = wisski.Metadata().Get(KeyLastRebuild, &epoch)
+	if err == ErrMetadatumNotSet {
+		return t, nil
+	}
+	if err != nil {
+		return t, err
+	}
+
+	// and turn it into time!
+	return time.Unix(epoch, 0), nil
+}
+
+func (wisski *WissKI) setLastRebuild() error {
+	return wisski.Metadata().Set(KeyLastRebuild, time.Now().Unix())
+}
+
+// Build builds or rebuilds the barel connected to this instance.
+//
+// It also logs the current time into the metadata belonging to this instance.
+func (wisski *WissKI) Build(stream stream.IOStream, start bool) error {
+	barrel := wisski.Barrel()
+
+	var context component.InstallationContext
+
+	{
+		err := barrel.Install(stream, context)
+		if err != nil {
+			return err
+		}
+	}
+
+	{
+		err := barrel.Update(stream, start)
+		if err != nil {
+			return err
+		}
+	}
+
+	// store the current last rebuild
+	return wisski.setLastRebuild()
+}
+
 //go:embed all:instances/reserve instances/reserve.env
 var reserveResources embed.FS
 
 // Reserve returns a stack representing the reserve instance
-func (wisski WissKI) Reserve() component.StackWithResources {
+func (wisski *WissKI) Reserve() component.StackWithResources {
 	return component.StackWithResources{
 		Stack: component.Stack{
 			Dir: wisski.FilesystemBase,

internal/component/instances/wisski_status.go

@@ -1,6 +1,9 @@
 package instances
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/sync/errgroup"
 )
@@ -10,12 +13,15 @@ type Info struct {
 	Slug string // The slug of the instance
 	URL  string // The public URL of this instance
 
+	LastRebuild time.Time
+
 	Running      bool     // is the instance running?
 	Pathbuilders []string // list of pathbuilders
 }
 
 // Info returns information about this WissKI instance.
 func (wisski *WissKI) Info(quick bool) (info Info, err error) {
+	fmt.Println("call to info")
 	// static properties
 	info.Slug = wisski.Slug
 	info.URL = wisski.URL().String()
@@ -30,15 +36,20 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) {
 	})
 
 	// slower checks for extra properties.
-	// these execute php code
+	// these might execute php code or require additional database queries.
 	if !quick {
+		group.Go(func() error {
+			info.Pathbuilders, _ = wisski.Pathbuilders()
+			return nil
+		})
 		group.Go(func() (err error) {
-			info.Pathbuilders, err = wisski.Pathbuilders()
-			return
+			info.LastRebuild, _ = wisski.LastRebuild()
+			return nil
 		})
 	}
 
 	err = group.Wait()
+	fmt.Println(err)
 	return
 }
 

internal/component/sql/connect.go

@@ -21,8 +21,8 @@ import (
 // ========== low-level connection ==========
 //
 
-// Query performs a database query, outside a database contect
-func (sql *SQL) Query(query string, args ...interface{}) error {
+// Exec executes a database-independent database query.
+func (sql *SQL) Exec(query string, args ...interface{}) error {
 	// connect to the server
 	conn, err := sql.connect("")
 	if err != nil {
@@ -39,10 +39,10 @@ func (sql *SQL) Query(query string, args ...interface{}) error {
 	}
 }
 
-// WaitQuery waits for the query interface to be able to connect to the database
-func (sql *SQL) WaitQuery() error {
+// WaitExec waits for the query interface to be able to connect to the database
+func (sql *SQL) WaitExec() error {
 	return wait.Wait(func() bool {
-		err := sql.Query("select 1;")
+		err := sql.Exec("select 1;")
 		// log.Printf("[WaitQuery] %s\n", err) // debug
 		return err == nil
 	}, sql.PollInterval, sql.PollContext)
@@ -52,8 +52,8 @@ func (sql *SQL) WaitQuery() error {
 // ========== connection via gorm ==========
 //
 
-// QueryTable returns a gorm.DB to connect to the provided gorm database table
-func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
+// QueryTable returns a gorm.DB to connect to the provided distillery database table
+func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
 	conn, err := sql.connect(sql.Config.DistilleryDatabase)
 	if err != nil {
 		return nil, err
@@ -79,7 +79,7 @@ func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
 	}
 
 	// set the table
-	db = db.Table(name)
+	db = db.Table(table)
 
 	// check that nothing went wrong
 	if db.Error != nil {

internal/component/sql/connect_shell.go

@@ -1 +0,0 @@
-package sql

internal/component/sql/provision.go

@@ -9,7 +9,10 @@ import (
 var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters")
 var errProvisionInvalidGrant = errors.New("Provision: Grant failed")
 
-// Provision provisions a new sql database and user
+// Provision creates a new database with the given name.
+// It then generates a new user, with the name 'user' and the password 'password', that is then granted access to this database.
+//
+// Provision internally waits for the database to become available.
 func (sql *SQL) Provision(name, user, password string) error {
 
 	// NOTE(twiesing): We shouldn't use string concat to build sql queries.
@@ -41,6 +44,10 @@ func (sql *SQL) Provision(name, user, password string) error {
 
 var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed")
 
+// CreateSuperuser createsa new user, with the name 'user' and the password 'password'.
+// It then grants this user superuser status in the database.
+//
+// CreateSuperuser internally waits for the database to become available.
 func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error {
 	// NOTE(twiesing): This function unsafely uses the shell directly to create a superuser.
 	// This is for two reasons:
@@ -70,9 +77,21 @@ func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error
 	return nil
 }
 
+var errPurgeUser = errors.New("PurgeUser: Failed to drop user")
+
 // SQLPurgeUser deletes the specified user from the database
 func (sql *SQL) PurgeUser(user string) error {
-	return sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user)
+	if !sqle.IsSafeDatabaseSingleQuote(user) {
+		return errPurgeUser
+	}
+
+	query := "DROP USER IF EXISTS '" + user + "'@'%';" +
+		"FLUSH PRIVILEGES;"
+	if !sql.unsafeQueryShell(query) {
+		return errPurgeUser
+	}
+
+	return nil
 }
 
 var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name")
@@ -82,5 +101,5 @@ func (sql *SQL) PurgeDatabase(db string) error {
 	if !sqle.IsSafeDatabaseLiteral(db) {
 		return errSQLPurgeDB
 	}
-	return sql.Query("DROP DATABASE IF EXISTS `" + db + "`")
+	return sql.Exec("DROP DATABASE IF EXISTS `" + db + "`")
 }

internal/component/sql/update.go

@@ -8,6 +8,7 @@ import (
 	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
 	"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
 	"github.com/FAU-CDI/wisski-distillery/pkg/wait"
+	"github.com/tkw1536/goprogram/exit"
 	"github.com/tkw1536/goprogram/stream"
 )
 
@@ -36,6 +37,10 @@ func (sql *SQL) unsafeQueryShell(query string) bool {
 
 var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
 var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name")
+var errSQLUnableToMigrate = exit.Error{
+	Message:  "unable to migrate %s table: %s",
+	ExitCode: exit.ExitGeneric,
+}
 
 // Update initializes or updates the SQL database.
 func (sql *SQL) Update(io stream.IOStream) error {
@@ -62,7 +67,7 @@ func (sql *SQL) Update(io stream.IOStream) error {
 			return errSQLUnsafeDatabaseName
 		}
 		createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase)
-		if err := sql.Query(createDBSQL); err != nil {
+		if err := sql.Exec(createDBSQL); err != nil {
 			return err
 		}
 	}
@@ -71,18 +76,36 @@ func (sql *SQL) Update(io stream.IOStream) error {
 	logging.LogMessage(io, "Waiting for database update to be complete")
 	sql.WaitQueryTable()
 
-	// open the database
-	logging.LogMessage(io, "Migrating instances table")
-	{
-		db, err := sql.QueryTable(false, models.InstanceTable)
-		if err != nil {
-			return fmt.Errorf("unable to access bookkeeping table: %s", err)
-		}
-
-		if err := db.AutoMigrate(&models.Instance{}); err != nil {
-			return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
-		}
+	tables := []struct {
+		name  string
+		model any
+		table string
+	}{
+		{
+			"instance",
+			&models.Instance{},
+			models.InstanceTable,
+		},
+		{
+			"metadata",
+			&models.Metadatum{},
+			models.MetadataTable,
+		},
 	}
 
-	return nil
+	// migrate all of the tables!
+	return logging.LogOperation(func() error {
+		for _, table := range tables {
+			logging.LogMessage(io, "migrating %q table", table.name)
+			db, err := sql.QueryTable(false, table.table)
+			if err != nil {
+				return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
+			}
+
+			if err := db.AutoMigrate(table.model); err != nil {
+				return errSQLUnableToMigrate.WithMessageF(table.name, err)
+			}
+		}
+		return nil
+	}, io, "migrating database tables")
 }

internal/component/stack.go

@@ -217,7 +217,8 @@ type InstallationContext map[string]string
 //
 // Installation is non-interactive, but will provide debugging output onto io.
 // InstallationContext
-func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
+func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
+	env := is.Stack.Env
 	if is.ContextPath != "" {
 		// setup the base files
 		if err := unpack.InstallDir(

internal/models/metadata.go

@@ -0,0 +1,13 @@
+package models
+
+// MetadataTable is the name of the table the 'Metadatum' model is stored in.
+const MetadataTable = "metadatum"
+
+// Metadatum represents
+type Metadatum struct {
+	Pk uint `gorm:"column:pk;primaryKey"`
+
+	Key   string `gorm:"column:key;not null"` // key for the value, see the keys below
+	Slug  string `gorm:"column:slug"`         // optional slug of instance
+	Value []byte `gorm:"column:value"`        // serialized json value of the data
+}

internal/wisski/snapshot.go

@@ -76,6 +76,7 @@ func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err er
 // SnapshotDescription is a description for a snapshot
 type SnapshotDescription struct {
 	Dest      string // destination path
+	Log       bool   // should we log the creation of this snapshot?
 	Keepalive bool   // should we keep the instance alive while making the snapshot?
 }