From 8b3218ad00c5ed88b79a1e5be467316002ebd4b4 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Tue, 20 Sep 2022 13:11:24 +0200 Subject: [PATCH] Add a metadata system --- cmd/purge.go | 5 + cmd/rebuild.go | 18 +- cmd/reserve.go | 2 +- cmd/system_update.go | 2 +- internal/component/control/html/instance.html | 1 + internal/component/instances/meta.go | 204 ++++++++++++++++++ internal/component/instances/wisski_create.go | 15 +- internal/component/instances/wisski_stack.go | 54 ++++- internal/component/instances/wisski_status.go | 17 +- internal/component/sql/connect.go | 16 +- internal/component/sql/connect_shell.go | 1 - internal/component/sql/provision.go | 25 ++- internal/component/sql/update.go | 49 +++-- internal/component/stack.go | 3 +- internal/models/metadata.go | 13 ++ internal/wisski/snapshot.go | 1 + 16 files changed, 365 insertions(+), 61 deletions(-) create mode 100644 internal/component/instances/meta.go delete mode 100644 internal/component/sql/connect_shell.go create mode 100644 internal/models/metadata.go diff --git a/cmd/purge.go b/cmd/purge.go index 39f6faf..7183fbf 100644 --- a/cmd/purge.go +++ b/cmd/purge.go @@ -114,6 +114,11 @@ func (p purge) Run(context wisski_distillery.Context) error { context.EPrintln(err) } + logging.LogMessage(context.IOStream, "Purging instance metadata") + if err := instance.Metadata().Purge(); err != nil { + context.EPrintln(err) + } + logging.LogMessage(context.IOStream, "Instance %s has been purged", slug) return nil } diff --git a/cmd/rebuild.go b/cmd/rebuild.go index 8dbbfba..2e86974 100644 --- a/cmd/rebuild.go +++ b/cmd/rebuild.go @@ -2,7 +2,6 @@ package cmd import ( wisski_distillery "github.com/FAU-CDI/wisski-distillery" - "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/internal/core" "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/tkw1536/goprogram/exit" @@ -44,22 +43,7 @@ func (rb rebuild) Run(context wisski_distillery.Context) error { var globalErr error for _, instance := range instances { logging.LogOperation(func() error { - s := instance.Barrel() - if err := logging.LogOperation(func() error { - return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{}) - }, context.IOStream, "Installing docker stack"); err != nil { - globalErr = err - return err - } - - if err := logging.LogOperation(func() error { - return s.Update(context.IOStream, true) - }, context.IOStream, "Updating docker stack"); err != nil { - globalErr = err - return err - } - - return nil + return instance.Build(context.IOStream, true) }, context.IOStream, "Rebuilding instance %s", instance.Slug) } diff --git a/cmd/reserve.go b/cmd/reserve.go index 40fd05d..4477b64 100644 --- a/cmd/reserve.go +++ b/cmd/reserve.go @@ -66,7 +66,7 @@ func (r reserve) Run(context wisski_distillery.Context) error { s := instance.Reserve() { if err := logging.LogOperation(func() error { - return s.Install(dis.Core.Environment, context.IOStream, component.InstallationContext{}) + return s.Install(context.IOStream, component.InstallationContext{}) }, context.IOStream, "Installing docker stack"); err != nil { return err } diff --git a/cmd/system_update.go b/cmd/system_update.go index 84c837e..a9801b4 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -112,7 +112,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error { ctx := component.Context(ctx) if err := logging.LogOperation(func() error { - return stack.Install(dis.Core.Environment, context.IOStream, ctx) + return stack.Install(context.IOStream, ctx) }, context.IOStream, "Installing Docker Stack %q", name); err != nil { return err } diff --git a/internal/component/control/html/instance.html b/internal/component/control/html/instance.html index e8c5d42..e5cbae1 100644 --- a/internal/component/control/html/instance.html +++ b/internal/component/control/html/instance.html @@ -13,6 +13,7 @@ URL: {{ .Info.URL }}
Running: {{ .Info.Running }}
+ Last Rebuild: {{ .Info.LastRebuild }}
Created: {{ .Instance.Created }}
OwnerEmail: {{ .Instance.OwnerEmail }}
diff --git a/internal/component/instances/meta.go b/internal/component/instances/meta.go new file mode 100644 index 0000000..d1aa9e3 --- /dev/null +++ b/internal/component/instances/meta.go @@ -0,0 +1,204 @@ +package instances + +import ( + "encoding/json" + "errors" + + "github.com/FAU-CDI/wisski-distillery/internal/component/sql" + "github.com/FAU-CDI/wisski-distillery/internal/models" + "gorm.io/gorm" +) + +// MetaKey represents a key for metadata. +type MetaKey string + +// ErrMetadatumNotSet is returned by various [MetaStorage] functions when a metadatum is not set +var ErrMetadatumNotSet = errors.New("metadatum not set") + +// MetaStorage manages some metadata. +type MetaStorage interface { + // Get retrieves metadata with the provided key and deserializes the first one into target. + // If no metadatum exists, returns [ErrMetadatumNotSet]. + Get(key MetaKey, target any) error + + // GetAll receives all metadata with the provided keys. + // For each received value, the targets function is called with the current index, and total number of results. + // The function is intended to return a target for deserialization. + // + // When no metadatum exists, targets is not called, and nil error is returned. + GetAll(key MetaKey, targets func(index, total int) any) error + + // Delete deletes all metadata with the provided key. + Delete(key MetaKey) error + + // Set serializes value and stores it with the provided key. + // Any other metadata with the same key is deleted. + Set(key MetaKey, value any) error + + // Add serializes values and stores each as associated with the provided key. + // Already existing metadata is left intact. + Add(key MetaKey, values ...any) error + + // Purge removes all metadata, regardless of key. + Purge() error +} + +// Metadata returns a system-wide [MetaStorage]. +func (instances *Instances) Metadata() MetaStorage { + return &storage{ + SQL: instances.SQL, + Slug: "", // not associated to any slug + } +} + +// Metadata returns a [MetaStorage] that manages metadata related to this WissKI instance. +// It will be automatically deleted once the instance is deleted. +func (wisski *WissKI) Metadata() MetaStorage { + return &storage{ + SQL: wisski.instances.SQL, + Slug: wisski.Slug, // associated to this instance + } +} + +// storage implements MetaStorage +type storage struct { + SQL *sql.SQL + Slug string +} + +func (s *storage) Get(key MetaKey, target any) error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + // read the datum from the database + var datum models.Metadatum + status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Order("pk DESC").Find(&datum) + + // check if there was an error + if err := status.Error; err != nil { + return err + } + + // check if e actually found it! + if status.RowsAffected == 0 { + return ErrMetadatumNotSet + } + + // and do the unmarshaling! + return json.Unmarshal(datum.Value, target) +} + +func (s *storage) GetAll(key MetaKey, target func(index, total int) any) error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + // read the datum from the database + var data []models.Metadatum + status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Find(&data) + + // check if there was an error + if err := status.Error; err != nil { + return err + } + + // unpack all of them into the destination + for index, datum := range data { + err := json.Unmarshal(datum.Value, target(index, len(data))) + if err != nil { + return err + } + } + + return nil +} + +func (s *storage) Delete(key MetaKey) error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + // delete all the values + status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{}) + if err := status.Error; err != nil { + return err + } + return nil +} + +func (s *storage) Set(key MetaKey, value any) error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + // marshal the value + bytes, err := json.Marshal(value) + if err != nil { + return err + } + + return table.Transaction(func(tx *gorm.DB) error { + // delete the old values + status := tx.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{}) + if err := status.Error; err != nil { + return err + } + + // create the new item to insert + status = tx.Create(&models.Metadatum{ + Key: string(key), + Slug: s.Slug, + Value: bytes, + }) + if err := status.Error; err != nil { + return err + } + + return nil + }) +} + +func (s *storage) Add(key MetaKey, values ...any) error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + return table.Transaction(func(tx *gorm.DB) error { + for _, value := range values { + bytes, err := json.Marshal(value) + if err != nil { + return err + } + + // create the new item to insert + status := tx.Create(&models.Metadatum{ + Key: string(key), + Slug: s.Slug, + Value: bytes, + }) + if err := status.Error; err != nil { + return err + } + } + return nil + }) +} + +func (s *storage) Purge() error { + table, err := s.SQL.QueryTable(true, models.MetadataTable) + if err != nil { + return err + } + + status := table.Where("slug = ?", s.Slug).Delete(&models.Metadatum{}) + if status.Error != nil { + return status.Error + } + return nil +} diff --git a/internal/component/instances/wisski_create.go b/internal/component/instances/wisski_create.go index e8f8ff7..af1c75d 100644 --- a/internal/component/instances/wisski_create.go +++ b/internal/component/instances/wisski_create.go @@ -5,7 +5,6 @@ import ( "path/filepath" "strings" - "github.com/FAU-CDI/wisski-distillery/internal/component" "github.com/FAU-CDI/wisski-distillery/pkg/stringparser" "github.com/alessio/shellescape" "github.com/tkw1536/goprogram/stream" @@ -66,16 +65,10 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) { } // Provision provisions an instance, assuming that the required databases already exist. -func (wisski WissKI) Provision(io stream.IOStream) error { +func (wisski *WissKI) Provision(io stream.IOStream) error { - // create the basic st! - st := wisski.Barrel() - if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil { - return err - } - - // Pull and build the stack! - if err := st.Update(io, false); err != nil { + // build the container + if err := wisski.Build(io, false); err != nil { return err } @@ -106,7 +99,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error { // TODO: Move the provision script into the control plane! provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ") - code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) + code, err := wisski.Barrel().Run(io, true, "barrel", "/bin/bash", "-c", provisionScript) if err != nil { return err } diff --git a/internal/component/instances/wisski_stack.go b/internal/component/instances/wisski_stack.go index 461c68b..48f7bbf 100644 --- a/internal/component/instances/wisski_stack.go +++ b/internal/component/instances/wisski_stack.go @@ -3,15 +3,17 @@ package instances import ( "embed" "path/filepath" + "time" "github.com/FAU-CDI/wisski-distillery/internal/component" + "github.com/tkw1536/goprogram/stream" ) //go:embed all:instances/barrel instances/barrel.env var barrelResources embed.FS // Barrel returns a stack representing the running WissKI Instance -func (wisski WissKI) Barrel() component.StackWithResources { +func (wisski *WissKI) Barrel() component.StackWithResources { return component.StackWithResources{ Stack: component.Stack{ Dir: wisski.FilesystemBase, @@ -43,11 +45,59 @@ func (wisski WissKI) Barrel() component.StackWithResources { } } +const KeyLastRebuild MetaKey = "lastRebuild" + +func (wisski *WissKI) LastRebuild() (t time.Time, err error) { + var epoch int64 + + // read the epoch! + err = wisski.Metadata().Get(KeyLastRebuild, &epoch) + if err == ErrMetadatumNotSet { + return t, nil + } + if err != nil { + return t, err + } + + // and turn it into time! + return time.Unix(epoch, 0), nil +} + +func (wisski *WissKI) setLastRebuild() error { + return wisski.Metadata().Set(KeyLastRebuild, time.Now().Unix()) +} + +// Build builds or rebuilds the barel connected to this instance. +// +// It also logs the current time into the metadata belonging to this instance. +func (wisski *WissKI) Build(stream stream.IOStream, start bool) error { + barrel := wisski.Barrel() + + var context component.InstallationContext + + { + err := barrel.Install(stream, context) + if err != nil { + return err + } + } + + { + err := barrel.Update(stream, start) + if err != nil { + return err + } + } + + // store the current last rebuild + return wisski.setLastRebuild() +} + //go:embed all:instances/reserve instances/reserve.env var reserveResources embed.FS // Reserve returns a stack representing the reserve instance -func (wisski WissKI) Reserve() component.StackWithResources { +func (wisski *WissKI) Reserve() component.StackWithResources { return component.StackWithResources{ Stack: component.Stack{ Dir: wisski.FilesystemBase, diff --git a/internal/component/instances/wisski_status.go b/internal/component/instances/wisski_status.go index 46eea7e..00827d8 100644 --- a/internal/component/instances/wisski_status.go +++ b/internal/component/instances/wisski_status.go @@ -1,6 +1,9 @@ package instances import ( + "fmt" + "time" + "github.com/tkw1536/goprogram/stream" "golang.org/x/sync/errgroup" ) @@ -10,12 +13,15 @@ type Info struct { Slug string // The slug of the instance URL string // The public URL of this instance + LastRebuild time.Time + Running bool // is the instance running? Pathbuilders []string // list of pathbuilders } // Info returns information about this WissKI instance. func (wisski *WissKI) Info(quick bool) (info Info, err error) { + fmt.Println("call to info") // static properties info.Slug = wisski.Slug info.URL = wisski.URL().String() @@ -30,15 +36,20 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) { }) // slower checks for extra properties. - // these execute php code + // these might execute php code or require additional database queries. if !quick { + group.Go(func() error { + info.Pathbuilders, _ = wisski.Pathbuilders() + return nil + }) group.Go(func() (err error) { - info.Pathbuilders, err = wisski.Pathbuilders() - return + info.LastRebuild, _ = wisski.LastRebuild() + return nil }) } err = group.Wait() + fmt.Println(err) return } diff --git a/internal/component/sql/connect.go b/internal/component/sql/connect.go index 35afe33..9c83136 100644 --- a/internal/component/sql/connect.go +++ b/internal/component/sql/connect.go @@ -21,8 +21,8 @@ import ( // ========== low-level connection ========== // -// Query performs a database query, outside a database contect -func (sql *SQL) Query(query string, args ...interface{}) error { +// Exec executes a database-independent database query. +func (sql *SQL) Exec(query string, args ...interface{}) error { // connect to the server conn, err := sql.connect("") if err != nil { @@ -39,10 +39,10 @@ func (sql *SQL) Query(query string, args ...interface{}) error { } } -// WaitQuery waits for the query interface to be able to connect to the database -func (sql *SQL) WaitQuery() error { +// WaitExec waits for the query interface to be able to connect to the database +func (sql *SQL) WaitExec() error { return wait.Wait(func() bool { - err := sql.Query("select 1;") + err := sql.Exec("select 1;") // log.Printf("[WaitQuery] %s\n", err) // debug return err == nil }, sql.PollInterval, sql.PollContext) @@ -52,8 +52,8 @@ func (sql *SQL) WaitQuery() error { // ========== connection via gorm ========== // -// QueryTable returns a gorm.DB to connect to the provided gorm database table -func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) { +// QueryTable returns a gorm.DB to connect to the provided distillery database table +func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) { conn, err := sql.connect(sql.Config.DistilleryDatabase) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) { } // set the table - db = db.Table(name) + db = db.Table(table) // check that nothing went wrong if db.Error != nil { diff --git a/internal/component/sql/connect_shell.go b/internal/component/sql/connect_shell.go deleted file mode 100644 index e4b317b..0000000 --- a/internal/component/sql/connect_shell.go +++ /dev/null @@ -1 +0,0 @@ -package sql diff --git a/internal/component/sql/provision.go b/internal/component/sql/provision.go index 6f147d6..912d21b 100644 --- a/internal/component/sql/provision.go +++ b/internal/component/sql/provision.go @@ -9,7 +9,10 @@ import ( var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters") var errProvisionInvalidGrant = errors.New("Provision: Grant failed") -// Provision provisions a new sql database and user +// Provision creates a new database with the given name. +// It then generates a new user, with the name 'user' and the password 'password', that is then granted access to this database. +// +// Provision internally waits for the database to become available. func (sql *SQL) Provision(name, user, password string) error { // NOTE(twiesing): We shouldn't use string concat to build sql queries. @@ -41,6 +44,10 @@ func (sql *SQL) Provision(name, user, password string) error { var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed") +// CreateSuperuser createsa new user, with the name 'user' and the password 'password'. +// It then grants this user superuser status in the database. +// +// CreateSuperuser internally waits for the database to become available. func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error { // NOTE(twiesing): This function unsafely uses the shell directly to create a superuser. // This is for two reasons: @@ -70,9 +77,21 @@ func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error return nil } +var errPurgeUser = errors.New("PurgeUser: Failed to drop user") + // SQLPurgeUser deletes the specified user from the database func (sql *SQL) PurgeUser(user string) error { - return sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) + if !sqle.IsSafeDatabaseSingleQuote(user) { + return errPurgeUser + } + + query := "DROP USER IF EXISTS '" + user + "'@'%';" + + "FLUSH PRIVILEGES;" + if !sql.unsafeQueryShell(query) { + return errPurgeUser + } + + return nil } var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name") @@ -82,5 +101,5 @@ func (sql *SQL) PurgeDatabase(db string) error { if !sqle.IsSafeDatabaseLiteral(db) { return errSQLPurgeDB } - return sql.Query("DROP DATABASE IF EXISTS `" + db + "`") + return sql.Exec("DROP DATABASE IF EXISTS `" + db + "`") } diff --git a/internal/component/sql/update.go b/internal/component/sql/update.go index 9372daf..0cdbc7c 100644 --- a/internal/component/sql/update.go +++ b/internal/component/sql/update.go @@ -8,6 +8,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/sqle" "github.com/FAU-CDI/wisski-distillery/pkg/wait" + "github.com/tkw1536/goprogram/exit" "github.com/tkw1536/goprogram/stream" ) @@ -36,6 +37,10 @@ func (sql *SQL) unsafeQueryShell(query string) bool { var errSQLUnableToCreateUser = errors.New("unable to create administrative user") var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name") +var errSQLUnableToMigrate = exit.Error{ + Message: "unable to migrate %s table: %s", + ExitCode: exit.ExitGeneric, +} // Update initializes or updates the SQL database. func (sql *SQL) Update(io stream.IOStream) error { @@ -62,7 +67,7 @@ func (sql *SQL) Update(io stream.IOStream) error { return errSQLUnsafeDatabaseName } createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase) - if err := sql.Query(createDBSQL); err != nil { + if err := sql.Exec(createDBSQL); err != nil { return err } } @@ -71,18 +76,36 @@ func (sql *SQL) Update(io stream.IOStream) error { logging.LogMessage(io, "Waiting for database update to be complete") sql.WaitQueryTable() - // open the database - logging.LogMessage(io, "Migrating instances table") - { - db, err := sql.QueryTable(false, models.InstanceTable) - if err != nil { - return fmt.Errorf("unable to access bookkeeping table: %s", err) - } - - if err := db.AutoMigrate(&models.Instance{}); err != nil { - return fmt.Errorf("unable to migrate bookkeeping table: %s", err) - } + tables := []struct { + name string + model any + table string + }{ + { + "instance", + &models.Instance{}, + models.InstanceTable, + }, + { + "metadata", + &models.Metadatum{}, + models.MetadataTable, + }, } - return nil + // migrate all of the tables! + return logging.LogOperation(func() error { + for _, table := range tables { + logging.LogMessage(io, "migrating %q table", table.name) + db, err := sql.QueryTable(false, table.table) + if err != nil { + return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table") + } + + if err := db.AutoMigrate(table.model); err != nil { + return errSQLUnableToMigrate.WithMessageF(table.name, err) + } + } + return nil + }, io, "migrating database tables") } diff --git a/internal/component/stack.go b/internal/component/stack.go index 809741b..8e0c1d5 100644 --- a/internal/component/stack.go +++ b/internal/component/stack.go @@ -217,7 +217,8 @@ type InstallationContext map[string]string // // Installation is non-interactive, but will provide debugging output onto io. // InstallationContext -func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error { +func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error { + env := is.Stack.Env if is.ContextPath != "" { // setup the base files if err := unpack.InstallDir( diff --git a/internal/models/metadata.go b/internal/models/metadata.go new file mode 100644 index 0000000..955bfcc --- /dev/null +++ b/internal/models/metadata.go @@ -0,0 +1,13 @@ +package models + +// MetadataTable is the name of the table the 'Metadatum' model is stored in. +const MetadataTable = "metadatum" + +// Metadatum represents +type Metadatum struct { + Pk uint `gorm:"column:pk;primaryKey"` + + Key string `gorm:"column:key;not null"` // key for the value, see the keys below + Slug string `gorm:"column:slug"` // optional slug of instance + Value []byte `gorm:"column:value"` // serialized json value of the data +} diff --git a/internal/wisski/snapshot.go b/internal/wisski/snapshot.go index dd13767..11bb39a 100644 --- a/internal/wisski/snapshot.go +++ b/internal/wisski/snapshot.go @@ -76,6 +76,7 @@ func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err er // SnapshotDescription is a description for a snapshot type SnapshotDescription struct { Dest string // destination path + Log bool // should we log the creation of this snapshot? Keepalive bool // should we keep the instance alive while making the snapshot? }