rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add a metadata system

Browse source
This commit is contained in:
Tom Wiesing 2022-09-20 13:11:24 +02:00
parent 07409a01be
commit 8b3218ad00
No known key found for this signature in database
16 changed files with 365 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

1
internal/component/control/html/instance.html
View file

@ -13,6 +13,7 @@
<b>URL:</b> <a href="{{ .Info.URL }}" target="_blank" rel="noopener noreferrer">{{ .Info.URL }}</a> <br />
<hr />
<b>Running:</b> <code>{{ .Info.Running }}</code> <br />
<b>Last Rebuild:</b> <code>{{ .Info.LastRebuild }}</code> <br />
<hr />
<b>Created:</b> <code>{{ .Instance.Created }}</code> <br />
<b>OwnerEmail:</b> <code>{{ .Instance.OwnerEmail }}</code> <br />

204
internal/component/instances/meta.go Normal file
View file

@ -0,0 +1,204 @@
package instances
import (
"encoding/json"
"errors"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"gorm.io/gorm"
)
// MetaKey represents a key for metadata.
type MetaKey string
// ErrMetadatumNotSet is returned by various [MetaStorage] functions when a metadatum is not set
var ErrMetadatumNotSet = errors.New("metadatum not set")
// MetaStorage manages some metadata.
type MetaStorage interface {
// Get retrieves metadata with the provided key and deserializes the first one into target.
// If no metadatum exists, returns [ErrMetadatumNotSet].
Get(key MetaKey, target any) error
// GetAll receives all metadata with the provided keys.
// For each received value, the targets function is called with the current index, and total number of results.
// The function is intended to return a target for deserialization.
//
// When no metadatum exists, targets is not called, and nil error is returned.
GetAll(key MetaKey, targets func(index, total int) any) error
// Delete deletes all metadata with the provided key.
Delete(key MetaKey) error
// Set serializes value and stores it with the provided key.
// Any other metadata with the same key is deleted.
Set(key MetaKey, value any) error
// Add serializes values and stores each as associated with the provided key.
// Already existing metadata is left intact.
Add(key MetaKey, values ...any) error
// Purge removes all metadata, regardless of key.
Purge() error
}
// Metadata returns a system-wide [MetaStorage].
func (instances *Instances) Metadata() MetaStorage {
return &storage{
SQL: instances.SQL,
Slug: "", // not associated to any slug
}
}
// Metadata returns a [MetaStorage] that manages metadata related to this WissKI instance.
// It will be automatically deleted once the instance is deleted.
func (wisski *WissKI) Metadata() MetaStorage {
return &storage{
SQL: wisski.instances.SQL,
Slug: wisski.Slug, // associated to this instance
}
}
// storage implements MetaStorage
type storage struct {
SQL *sql.SQL
Slug string
}
func (s *storage) Get(key MetaKey, target any) error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
// read the datum from the database
var datum models.Metadatum
status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Order("pk DESC").Find(&datum)
// check if there was an error
if err := status.Error; err != nil {
return err
}
// check if e actually found it!
if status.RowsAffected == 0 {
return ErrMetadatumNotSet
}
// and do the unmarshaling!
return json.Unmarshal(datum.Value, target)
}
func (s *storage) GetAll(key MetaKey, target func(index, total int) any) error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
// read the datum from the database
var data []models.Metadatum
status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Find(&data)
// check if there was an error
if err := status.Error; err != nil {
return err
}
// unpack all of them into the destination
for index, datum := range data {
err := json.Unmarshal(datum.Value, target(index, len(data)))
if err != nil {
return err
}
}
return nil
}
func (s *storage) Delete(key MetaKey) error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
// delete all the values
status := table.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
if err := status.Error; err != nil {
return err
}
return nil
}
func (s *storage) Set(key MetaKey, value any) error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
// marshal the value
bytes, err := json.Marshal(value)
if err != nil {
return err
}
return table.Transaction(func(tx *gorm.DB) error {
// delete the old values
status := tx.Where(&models.Metadatum{Slug: s.Slug, Key: string(key)}).Delete(&models.Metadatum{})
if err := status.Error; err != nil {
return err
}
// create the new item to insert
status = tx.Create(&models.Metadatum{
Key: string(key),
Slug: s.Slug,
Value: bytes,
})
if err := status.Error; err != nil {
return err
}
return nil
})
}
func (s *storage) Add(key MetaKey, values ...any) error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
return table.Transaction(func(tx *gorm.DB) error {
for _, value := range values {
bytes, err := json.Marshal(value)
if err != nil {
return err
}
// create the new item to insert
status := tx.Create(&models.Metadatum{
Key: string(key),
Slug: s.Slug,
Value: bytes,
})
if err := status.Error; err != nil {
return err
}
}
return nil
})
}
func (s *storage) Purge() error {
table, err := s.SQL.QueryTable(true, models.MetadataTable)
if err != nil {
return err
}
status := table.Where("slug = ?", s.Slug).Delete(&models.Metadatum{})
if status.Error != nil {
return status.Error
}
return nil
}

15
internal/component/instances/wisski_create.go
View file

@ -5,7 +5,6 @@ import (
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/stringparser"
"github.com/alessio/shellescape"
"github.com/tkw1536/goprogram/stream"
@ -66,16 +65,10 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
}
// Provision provisions an instance, assuming that the required databases already exist.
func (wisski WissKI) Provision(io stream.IOStream) error {
func (wisski *WissKI) Provision(io stream.IOStream) error {
// create the basic st!
st := wisski.Barrel()
if err := st.Install(wisski.instances.Core.Environment, io, component.InstallationContext{}); err != nil {
return err
}
// Pull and build the stack!
if err := st.Update(io, false); err != nil {
// build the container
if err := wisski.Build(io, false); err != nil {
return err
}
@ -106,7 +99,7 @@ func (wisski WissKI) Provision(io stream.IOStream) error {
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
code, err := wisski.Barrel().Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}

54
internal/component/instances/wisski_stack.go
View file

@ -3,15 +3,17 @@ package instances
import (
"embed"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/tkw1536/goprogram/stream"
)
//go:embed all:instances/barrel instances/barrel.env
var barrelResources embed.FS
// Barrel returns a stack representing the running WissKI Instance
func (wisski WissKI) Barrel() component.StackWithResources {
func (wisski *WissKI) Barrel() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,
@ -43,11 +45,59 @@ func (wisski WissKI) Barrel() component.StackWithResources {
}
}
const KeyLastRebuild MetaKey = "lastRebuild"
func (wisski *WissKI) LastRebuild() (t time.Time, err error) {
var epoch int64
// read the epoch!
err = wisski.Metadata().Get(KeyLastRebuild, &epoch)
if err == ErrMetadatumNotSet {
return t, nil
}
if err != nil {
return t, err
}
// and turn it into time!
return time.Unix(epoch, 0), nil
}
func (wisski *WissKI) setLastRebuild() error {
return wisski.Metadata().Set(KeyLastRebuild, time.Now().Unix())
}
// Build builds or rebuilds the barel connected to this instance.
//
// It also logs the current time into the metadata belonging to this instance.
func (wisski *WissKI) Build(stream stream.IOStream, start bool) error {
barrel := wisski.Barrel()
var context component.InstallationContext
{
err := barrel.Install(stream, context)
if err != nil {
return err
}
}
{
err := barrel.Update(stream, start)
if err != nil {
return err
}
}
// store the current last rebuild
return wisski.setLastRebuild()
}
//go:embed all:instances/reserve instances/reserve.env
var reserveResources embed.FS
// Reserve returns a stack representing the reserve instance
func (wisski WissKI) Reserve() component.StackWithResources {
func (wisski *WissKI) Reserve() component.StackWithResources {
return component.StackWithResources{
Stack: component.Stack{
Dir: wisski.FilesystemBase,

17
internal/component/instances/wisski_status.go
View file

@ -1,6 +1,9 @@
package instances
import (
"fmt"
"time"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
)
@ -10,12 +13,15 @@ type Info struct {
Slug string // The slug of the instance
URL string // The public URL of this instance
LastRebuild time.Time
Running bool // is the instance running?
Pathbuilders []string // list of pathbuilders
}
// Info returns information about this WissKI instance.
func (wisski *WissKI) Info(quick bool) (info Info, err error) {
fmt.Println("call to info")
// static properties
info.Slug = wisski.Slug
info.URL = wisski.URL().String()
@ -30,15 +36,20 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) {
})
// slower checks for extra properties.
// these execute php code
// these might execute php code or require additional database queries.
if !quick {
group.Go(func() error {
info.Pathbuilders, _ = wisski.Pathbuilders()
return nil
})
group.Go(func() (err error) {
info.Pathbuilders, err = wisski.Pathbuilders()
return
info.LastRebuild, _ = wisski.LastRebuild()
return nil
})
}
err = group.Wait()
fmt.Println(err)
return
}

16
internal/component/sql/connect.go
View file

@ -21,8 +21,8 @@ import (
// ========== low-level connection ==========
//
// Query performs a database query, outside a database contect
func (sql *SQL) Query(query string, args ...interface{}) error {
// Exec executes a database-independent database query.
func (sql *SQL) Exec(query string, args ...interface{}) error {
// connect to the server
conn, err := sql.connect("")
if err != nil {
@ -39,10 +39,10 @@ func (sql *SQL) Query(query string, args ...interface{}) error {
}
}
// WaitQuery waits for the query interface to be able to connect to the database
func (sql *SQL) WaitQuery() error {
// WaitExec waits for the query interface to be able to connect to the database
func (sql *SQL) WaitExec() error {
return wait.Wait(func() bool {
err := sql.Query("select 1;")
err := sql.Exec("select 1;")
// log.Printf("[WaitQuery] %s\n", err) // debug
return err == nil
}, sql.PollInterval, sql.PollContext)
@ -52,8 +52,8 @@ func (sql *SQL) WaitQuery() error {
// ========== connection via gorm ==========
//
// QueryTable returns a gorm.DB to connect to the provided gorm database table
func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
// QueryTable returns a gorm.DB to connect to the provided distillery database table
func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
conn, err := sql.connect(sql.Config.DistilleryDatabase)
if err != nil {
return nil, err
@ -79,7 +79,7 @@ func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
}
// set the table
db = db.Table(name)
db = db.Table(table)
// check that nothing went wrong
if db.Error != nil {

1
internal/component/sql/connect_shell.go
View file

@ -1 +0,0 @@
package sql

25
internal/component/sql/provision.go
View file

@ -9,7 +9,10 @@ import (
var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters")
var errProvisionInvalidGrant = errors.New("Provision: Grant failed")
// Provision provisions a new sql database and user
// Provision creates a new database with the given name.
// It then generates a new user, with the name 'user' and the password 'password', that is then granted access to this database.
//
// Provision internally waits for the database to become available.
func (sql *SQL) Provision(name, user, password string) error {
// NOTE(twiesing): We shouldn't use string concat to build sql queries.
@ -41,6 +44,10 @@ func (sql *SQL) Provision(name, user, password string) error {
var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed")
// CreateSuperuser createsa new user, with the name 'user' and the password 'password'.
// It then grants this user superuser status in the database.
//
// CreateSuperuser internally waits for the database to become available.
func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error {
// NOTE(twiesing): This function unsafely uses the shell directly to create a superuser.
// This is for two reasons:
@ -70,9 +77,21 @@ func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error
return nil
}
var errPurgeUser = errors.New("PurgeUser: Failed to drop user")
// SQLPurgeUser deletes the specified user from the database
func (sql *SQL) PurgeUser(user string) error {
return sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user)
if !sqle.IsSafeDatabaseSingleQuote(user) {
return errPurgeUser
}
query := "DROP USER IF EXISTS '" + user + "'@'%';" +
"FLUSH PRIVILEGES;"
if !sql.unsafeQueryShell(query) {
return errPurgeUser
}
return nil
}
var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name")
@ -82,5 +101,5 @@ func (sql *SQL) PurgeDatabase(db string) error {
if !sqle.IsSafeDatabaseLiteral(db) {
return errSQLPurgeDB
}
return sql.Query("DROP DATABASE IF EXISTS `" + db + "`")
return sql.Exec("DROP DATABASE IF EXISTS `" + db + "`")
}

49
internal/component/sql/update.go
View file

@ -8,6 +8,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/stream"
)
@ -36,6 +37,10 @@ func (sql *SQL) unsafeQueryShell(query string) bool {
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name")
var errSQLUnableToMigrate = exit.Error{
Message: "unable to migrate %s table: %s",
ExitCode: exit.ExitGeneric,
}
// Update initializes or updates the SQL database.
func (sql *SQL) Update(io stream.IOStream) error {
@ -62,7 +67,7 @@ func (sql *SQL) Update(io stream.IOStream) error {
return errSQLUnsafeDatabaseName
}
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase)
if err := sql.Query(createDBSQL); err != nil {
if err := sql.Exec(createDBSQL); err != nil {
return err
}
}
@ -71,18 +76,36 @@ func (sql *SQL) Update(io stream.IOStream) error {
logging.LogMessage(io, "Waiting for database update to be complete")
sql.WaitQueryTable()
// open the database
logging.LogMessage(io, "Migrating instances table")
{
db, err := sql.QueryTable(false, models.InstanceTable)
if err != nil {
return fmt.Errorf("unable to access bookkeeping table: %s", err)
}
if err := db.AutoMigrate(&models.Instance{}); err != nil {
return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
}
tables := []struct {
name string
model any
table string
}{
{
"instance",
&models.Instance{},
models.InstanceTable,
},
{
"metadata",
&models.Metadatum{},
models.MetadataTable,
},
}
return nil
// migrate all of the tables!
return logging.LogOperation(func() error {
for _, table := range tables {
logging.LogMessage(io, "migrating %q table", table.name)
db, err := sql.QueryTable(false, table.table)
if err != nil {
return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
}
if err := db.AutoMigrate(table.model); err != nil {
return errSQLUnableToMigrate.WithMessageF(table.name, err)
}
}
return nil
}, io, "migrating database tables")
}

3
internal/component/stack.go
View file

@ -217,7 +217,8 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is StackWithResources) Install(env environment.Environment, io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(io stream.IOStream, context InstallationContext) error {
env := is.Stack.Env
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(