env: Move each component into a separate struct

This commit cleans up the distillery code by making each component a
distinct struct. Each of these components is also returned by by a new
Component() function that replaces the Stacks() function.
This commit is contained in:
Tom Wiesing 2022-09-05 15:50:23 +02:00
parent 2a14d93d3c
commit 09431c4869
No known key found for this signature in database
16 changed files with 265 additions and 148 deletions

View file

@ -18,7 +18,6 @@ Work in progress.
## Future Work ## Future Work
- Move `provision_entrypoint.sh` into go - Move `provision_entrypoint.sh` into go
- Clean up the distillery code, by moving to seperate structs per component
- Rename backups to 'snapshots' and make them restorable - Rename backups to 'snapshots' and make them restorable
- Snapshot the docker images being used also! - Snapshot the docker images being used also!
- Avoid running `docker compose` executable and shift it to a library - Avoid running `docker compose` executable and shift it to a library

View file

@ -156,7 +156,7 @@ func (bi backupInstance) makeSnapshot(context wisski_distillery.Context, path st
defer nquads.Close() defer nquads.Close()
// TODO: Add a progress bar? // TODO: Add a progress bar?
_, err = dis.TriplestoreBackup(nquads, instance.GraphDBRepository) _, err = dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
return err return err
}, context.IOStream, "Backing up Triplestore"); err != nil { }, context.IOStream, "Backing up Triplestore"); err != nil {
return errBackupFailed.Wrap(err) return errBackupFailed.Wrap(err)
@ -175,7 +175,7 @@ func (bi backupInstance) makeSnapshot(context wisski_distillery.Context, path st
defer sql.Close() defer sql.Close()
// TODO: Add a progress bar? // TODO: Add a progress bar?
return dis.SQLBackup(context.IOStream, sql, instance.SqlDatabase) return dis.SQL().Backup(context.IOStream, sql, instance.SqlDatabase)
}, context.IOStream, "Backing up Triplestore"); err != nil { }, context.IOStream, "Backing up Triplestore"); err != nil {
return errBackupFailed.Wrap(err) return errBackupFailed.Wrap(err)
} }

View file

@ -55,7 +55,7 @@ func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error {
if err != nil { if err != nil {
return err return err
} }
code, err := context.Environment.SQLShell(context.IOStream, "-e", query) code, err := context.Environment.SQL().OpenShell(context.IOStream, "-e", query)
if err != nil { if err != nil {
return err return err
} }

View file

@ -32,7 +32,7 @@ func (mysql) Description() wisski_distillery.Description {
} }
func (ms mysql) Run(context wisski_distillery.Context) error { func (ms mysql) Run(context wisski_distillery.Context) error {
code, err := context.Environment.SQLShell(context.IOStream, ms.Positionals.Args...) code, err := context.Environment.SQL().OpenShell(context.IOStream, ms.Positionals.Args...)
if err != nil { if err != nil {
return err return err
} }

View file

@ -38,7 +38,8 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
return errPrefixUpdateFailed.WithMessageF(err) return errPrefixUpdateFailed.WithMessageF(err)
} }
target := dis.ResolverPrefixConfig() resolver := dis.Resolver()
target := resolver.ConfigPath()
// print the configuration // print the configuration
config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm) config, err := os.OpenFile(target, os.O_WRONLY, fs.ModePerm)
@ -69,7 +70,7 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
// and restart the resolver to apply the config! // and restart the resolver to apply the config!
logging.LogMessage(context.IOStream, "restarting resolver stack") logging.LogMessage(context.IOStream, "restarting resolver stack")
if err := dis.ResolverStack().Restart(context.IOStream); err != nil { if err := resolver.Stack().Restart(context.IOStream); err != nil {
return errPrefixUpdateFailed.WithMessageF(err) return errPrefixUpdateFailed.WithMessageF(err)
} }

View file

@ -74,7 +74,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
// create the sql // create the sql
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
if err := dis.SQLProvision(instance.SqlDatabase, instance.SqlUser, instance.SqlPassword); err != nil { if err := dis.SQL().Provision(instance.SqlDatabase, instance.SqlUser, instance.SqlPassword); err != nil {
return errProvisionGeneric.WithMessageF(slug, err) return errProvisionGeneric.WithMessageF(slug, err)
} }
@ -85,7 +85,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
// create the triplestore // create the triplestore
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
if err := dis.TriplestoreProvision(instance.GraphDBRepository, instance.Domain(), instance.GraphDBUser, instance.GraphDBPassword); err != nil { if err := dis.Triplestore().Provision(instance.GraphDBRepository, instance.Domain(), instance.GraphDBUser, instance.GraphDBPassword); err != nil {
return errProvisionGeneric.WithMessageF(slug, err) return errProvisionGeneric.WithMessageF(slug, err)
} }

View file

@ -77,14 +77,15 @@ func (p purge) Run(context wisski_distillery.Context) error {
} }
// remove the triplestore // remove the triplestore
ts := dis.Triplestore()
logging.LogOperation(func() error { logging.LogOperation(func() error {
logging.LogMessage(context.IOStream, "Removing user %s", instance.GraphDBUser) logging.LogMessage(context.IOStream, "Removing user %s", instance.GraphDBUser)
if err := dis.TriplestorePurgeUser(instance.GraphDBUser); err != nil { if err := ts.PurgeUser(instance.GraphDBUser); err != nil {
context.EPrintln(err) context.EPrintln(err)
} }
logging.LogMessage(context.IOStream, "Removing repository %s", instance.GraphDBRepository) logging.LogMessage(context.IOStream, "Removing repository %s", instance.GraphDBRepository)
if err := dis.TriplestorePurgeRepo(instance.GraphDBRepository); err != nil { if err := ts.PurgeRepo(instance.GraphDBRepository); err != nil {
context.EPrintln(err) context.EPrintln(err)
} }
@ -93,13 +94,15 @@ func (p purge) Run(context wisski_distillery.Context) error {
// remove the sql // remove the sql
logging.LogOperation(func() error { logging.LogOperation(func() error {
sql := dis.SQL()
logging.LogMessage(context.IOStream, "Removing user %s", instance.SqlUser) logging.LogMessage(context.IOStream, "Removing user %s", instance.SqlUser)
if err := dis.SQLPurgeUser(instance.SqlUser); err != nil { if err := sql.PurgeUser(instance.SqlUser); err != nil {
context.EPrintln(err) context.EPrintln(err)
} }
logging.LogMessage(context.IOStream, "Removing database %s", instance.SqlDatabase) logging.LogMessage(context.IOStream, "Removing database %s", instance.SqlDatabase)
if err := dis.SQLPurgeDatabase(instance.SqlDatabase); err != nil { if err := sql.PurgeUser(instance.SqlDatabase); err != nil {
context.EPrintln(err) context.EPrintln(err)
} }

View file

@ -122,16 +122,17 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
} }
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
for _, stack := range dis.Stacks() { for _, component := range dis.Components() {
stack := component.Stack()
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
return stack.Install(context.IOStream, ctx) return stack.Install(context.IOStream, ctx)
}, context.IOStream, "Installing docker stack %q", stack.Dir); err != nil { }, context.IOStream, "Installing docker stack %q", component.Name()); err != nil {
return err return err
} }
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
return stack.Update(context.IOStream, true) return stack.Update(context.IOStream, true)
}, context.IOStream, "Updating docker stack %q", stack.Dir); err != nil { }, context.IOStream, "Updating docker stack %q", component.Name()); err != nil {
return err return err
} }
} }
@ -149,13 +150,13 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
} }
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
return dis.SQLBootstrap(context.IOStream) return dis.SQL().Bootstrap(context.IOStream)
}, context.IOStream, "Bootstraping SQL database"); err != nil { }, context.IOStream, "Bootstraping SQL database"); err != nil {
return errBootstrapSQL.WithMessageF(err) return errBootstrapSQL.WithMessageF(err)
} }
if err := logging.LogOperation(func() error { if err := logging.LogOperation(func() error {
return dis.TriplestoreBootstrap(context.IOStream) return dis.Triplestore().Bootstrap(context.IOStream)
}, context.IOStream, "Bootstraping Triplestore"); err != nil { }, context.IOStream, "Bootstraping Triplestore"); err != nil {
return errBootstrapTriplestore.WithMessageF(err) return errBootstrapTriplestore.WithMessageF(err)
} }

19
env/instances.go vendored
View file

@ -37,11 +37,12 @@ var errSQL = exit.Error{
// Instance returns the instance of the WissKI Distillery with the provided slug // Instance returns the instance of the WissKI Distillery with the provided slug
func (dis *Distillery) Instance(slug string) (i Instance, err error) { func (dis *Distillery) Instance(slug string) (i Instance, err error) {
if err := dis.SQLWaitForConnection(); err != nil { sql := dis.SQL()
if err := sql.Wait(); err != nil {
return i, err return i, err
} }
table, err := dis.sqlBkTable(false) table, err := sql.OpenBookkeeping(false)
if err != nil { if err != nil {
return i, err return i, err
} }
@ -61,11 +62,12 @@ func (dis *Distillery) Instance(slug string) (i Instance, err error) {
// HasInstance checks if the provided instance exists in the bookeeping table // HasInstance checks if the provided instance exists in the bookeeping table
func (dis *Distillery) HasInstance(slug string) (ok bool, err error) { func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
if err := dis.SQLWaitForConnection(); err != nil { sql := dis.SQL()
if err := sql.Wait(); err != nil {
return false, err return false, err
} }
table, err := dis.sqlBkTable(false) table, err := sql.OpenBookkeeping(false)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -104,12 +106,13 @@ func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
// findInstances finds instance objects based on a query in the bookkeeping table // findInstances finds instance objects based on a query in the bookkeeping table
func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) { func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
if err := dis.SQLWaitForConnection(); err != nil { sql := dis.SQL()
if err := sql.Wait(); err != nil {
return nil, err return nil, err
} }
// open the bookkeeping table // open the bookkeeping table
table, err := dis.sqlBkTable(false) table, err := sql.OpenBookkeeping(false)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -153,7 +156,7 @@ type Instance struct {
// Update updates the bookkeeping table with this instance. // Update updates the bookkeeping table with this instance.
func (instance *Instance) Update() error { func (instance *Instance) Update() error {
db, err := instance.dis.sqlBkTable(false) db, err := instance.dis.SQL().OpenBookkeeping(false)
if err != nil { if err != nil {
return err return err
} }
@ -169,7 +172,7 @@ func (instance *Instance) Update() error {
// Delete deletes this instance from the bookkeeping table // Delete deletes this instance from the bookkeeping table
func (instance *Instance) Delete() error { func (instance *Instance) Delete() error {
db, err := instance.dis.sqlBkTable(false) db, err := instance.dis.SQL().OpenBookkeeping(false)
if err != nil { if err != nil {
return err return err
} }

29
env/stack.go vendored
View file

@ -9,20 +9,31 @@ import (
// TODO: Move everything into specific subpackages // TODO: Move everything into specific subpackages
// Stacks returns the Stacks of this distillery // Stacks returns the Stacks of this distillery
func (dis *Distillery) Stacks() []stack.Installable { func (dis *Distillery) Components() []Component {
// TODO: Do we want to cache these stacks? // TODO: Do we want to cache these stacks?
return []stack.Installable{ return []Component{
dis.WebStack(), dis.Web(),
dis.SelfStack(), dis.Self(),
dis.ResolverStack(), dis.Resolver(),
dis.SSHStack(), dis.SSH(),
dis.TriplestoreStack(), dis.Triplestore(),
dis.SQLStack(), dis.SQL(),
} }
} }
// Component represents a component of the distillery
type Component interface {
Name() string // Name is the name of this component
Stack() stack.Installable // Stack returns the installable stack representing this component
Path() string // Path returns the path to this component
}
// asCoreStack treats the provided stack as a core component of this distillery. // asCoreStack treats the provided stack as a core component of this distillery.
func (dis *Distillery) asCoreStack(name string, stack stack.Installable) stack.Installable { func (dis *Distillery) makeComponentStack(component Component, stack stack.Installable) stack.Installable {
name := component.Name()
stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", name) stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", name)
stack.ContextResource = filepath.Join("resources", "compose", name) stack.ContextResource = filepath.Join("resources", "compose", name)

46
env/stack_resolver.go vendored
View file

@ -7,29 +7,47 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/stack" "github.com/FAU-CDI/wisski-distillery/internal/stack"
) )
const ResolverPrefixFile = "prefix.cfg" // ResolverComponent represents the 'resolver' layer belonging to a distillery
type ResolverComponent struct {
ConfigName string // Filename of the configuration file
func (dis *Distillery) ResolverStack() stack.Installable { dis *Distillery
stack := dis.asCoreStack("resolver", stack.Installable{ }
// Resolver returns the ResolverComponent belonging to this distillery
func (dis *Distillery) Resolver() ResolverComponent {
return ResolverComponent{
ConfigName: "prefix.cfg",
dis: dis,
}
}
func (ResolverComponent) Name() string {
return "resolver"
}
func (resolver ResolverComponent) Stack() stack.Installable {
stack := resolver.dis.makeComponentStack(resolver, stack.Installable{
EnvFileContext: map[string]string{ EnvFileContext: map[string]string{
"VIRTUAL_HOST": dis.DefaultVirtualHost(), "VIRTUAL_HOST": resolver.dis.DefaultVirtualHost(),
"LETSENCRYPT_HOST": dis.DefaultLetsencryptHost(), "LETSENCRYPT_HOST": resolver.dis.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail, "LETSENCRYPT_EMAIL": resolver.dis.Config.CertbotEmail,
"PREFIX_FILE": "", // set below! "PREFIX_FILE": "", // set below!
"DEFAULT_DOMAIN": dis.Config.DefaultDomain, "DEFAULT_DOMAIN": resolver.dis.Config.DefaultDomain,
"LEGACY_DOMAIN": strings.Join(dis.Config.SelfExtraDomains, ","), "LEGACY_DOMAIN": strings.Join(resolver.dis.Config.SelfExtraDomains, ","),
}, },
TouchFiles: []string{ResolverPrefixFile}, TouchFiles: []string{resolver.ConfigName},
}) })
stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, ResolverPrefixFile) stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, resolver.ConfigName)
return stack return stack
} }
func (dis *Distillery) ResolverStackPath() string { func (resolver ResolverComponent) Path() string {
return dis.ResolverStack().Dir return resolver.Stack().Dir
} }
func (dis Distillery) ResolverPrefixConfig() string { func (resolver ResolverComponent) ConfigPath() string {
return filepath.Join(dis.ResolverStackPath(), ResolverPrefixFile) return filepath.Join(resolver.Path(), resolver.ConfigName)
} }

34
env/stack_self.go vendored
View file

@ -2,23 +2,37 @@ package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack" import "github.com/FAU-CDI/wisski-distillery/internal/stack"
func (dis *Distillery) SelfStack() stack.Installable { // SelfComponent represents the 'self' layer belonging to a distillery
type SelfComponent struct {
dis *Distillery
}
// Self returns the SelfComponent belonging to this distillery
func (dis *Distillery) Self() SelfComponent {
return SelfComponent{dis: dis}
}
func (SelfComponent) Name() string {
return "self"
}
func (sc SelfComponent) Stack() stack.Installable {
TARGET := "https://github.com/FAU-CDI/wisski-distillery" TARGET := "https://github.com/FAU-CDI/wisski-distillery"
if dis.Config.SelfRedirect != nil { if sc.dis.Config.SelfRedirect != nil {
TARGET = dis.Config.SelfRedirect.String() TARGET = sc.dis.Config.SelfRedirect.String()
} }
return dis.asCoreStack("self", stack.Installable{ return sc.dis.makeComponentStack(sc, stack.Installable{
EnvFileContext: map[string]string{ EnvFileContext: map[string]string{
"VIRTUAL_HOST": dis.DefaultVirtualHost(), "VIRTUAL_HOST": sc.dis.DefaultVirtualHost(),
"LETSENCRYPT_HOST": dis.DefaultLetsencryptHost(), "LETSENCRYPT_HOST": sc.dis.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail, "LETSENCRYPT_EMAIL": sc.dis.Config.CertbotEmail,
"TARGET": TARGET, "TARGET": TARGET,
"OVERRIDES_FILE": dis.Config.SelfOverridesFile, "OVERRIDES_FILE": sc.dis.Config.SelfOverridesFile,
}, },
}) })
} }
func (dis *Distillery) SelfStackPath() string { func (sc SelfComponent) Path() string {
return dis.SelfStack().Dir return sc.Stack().Dir
} }

118
env/stack_sql.go vendored
View file

@ -18,9 +18,29 @@ import (
"gorm.io/gorm/logger" "gorm.io/gorm/logger"
) )
// SQLStack returns the docker stack that handles the sql database. // SQLComponent represents the 'sql' layer belonging to a distillery
func (dis *Distillery) SQLStack() stack.Installable { type SQLComponent struct {
return dis.asCoreStack("sql", stack.Installable{ PollInterval time.Duration // Duration to wait for during wait
dis *Distillery
}
// SSH returns the SSHComponent belonging to this distillery
func (dis *Distillery) SQL() SQLComponent {
return SQLComponent{
PollInterval: time.Second,
dis: dis,
}
}
func (SQLComponent) Name() string {
return "sql"
}
// Stack returns the docker stack that handles the sql database.
func (sql SQLComponent) Stack() stack.Installable {
return sql.dis.makeComponentStack(sql, stack.Installable{
MakeDirsPerm: fs.ModeDir | fs.ModePerm, MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{ MakeDirs: []string{
"data", "data",
@ -29,18 +49,18 @@ func (dis *Distillery) SQLStack() stack.Installable {
} }
// SQLStackPath returns the path the SQLStack() lives at. // SQLStackPath returns the path the SQLStack() lives at.
func (dis *Distillery) SQLStackPath() string { func (sql SQLComponent) Path() string {
return dis.SQLStack().Dir return sql.Stack().Dir
} }
// sqlOpen opens a new sql connection to the provided database using the administrative credentials // sqlOpen opens a new sql connection to the provided database using the administrative credentials
func (env Distillery) sqlOpen(database string, config *gorm.Config) (*gorm.DB, error) { func (sql SQLComponent) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
sql := mysql.Config{ cfg := mysql.Config{
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", env.Config.MysqlAdminUser, env.Config.MysqlAdminPassword, "127.0.0.1:3306", database), DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.dis.Config.MysqlAdminUser, sql.dis.Config.MysqlAdminPassword, "127.0.0.1:3306", database),
DefaultStringSize: 256, DefaultStringSize: 256,
} }
db, err := gorm.Open(mysql.New(sql), config) db, err := gorm.Open(mysql.New(cfg), config)
if err != nil { if err != nil {
return db, err return db, err
} }
@ -54,8 +74,8 @@ func (env Distillery) sqlOpen(database string, config *gorm.Config) (*gorm.DB, e
return db, nil return db, nil
} }
// sqlBkTable returns a gorm connection to the bookkeeping database. // OpenBookkeeping opens a connection to the bookkeeping database
func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) { func (sql SQLComponent) OpenBookkeeping(silent bool) (*gorm.DB, error) {
config := &gorm.Config{} config := &gorm.Config{}
if silent { if silent {
@ -63,13 +83,13 @@ func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) {
} }
// open the database // open the database
db, err := dis.sqlOpen(dis.Config.DistilleryBookkeepingDatabase, config) db, err := sql.openDatabase(sql.dis.Config.DistilleryBookkeepingDatabase, config)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// load the table // load the table
table := db.Table(dis.Config.DistilleryBookkeepingTable) table := db.Table(sql.dis.Config.DistilleryBookkeepingTable)
if table.Error != nil { if table.Error != nil {
return nil, err return nil, err
} }
@ -79,11 +99,11 @@ func (dis *Distillery) sqlBkTable(silent bool) (*gorm.DB, error) {
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code") var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
// SQLBackup makes a backup of the sql database into dest. // Backup makes a backup of the sql database into dest.
func (dis *Distillery) SQLBackup(io stream.IOStream, dest io.Writer, database string) error { func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0) io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := dis.SQLStack().Exec(io, "sql", "mysqldump", "--database", database) code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--database", database)
if err != nil { if err != nil {
return err return err
} }
@ -93,42 +113,40 @@ func (dis *Distillery) SQLBackup(io stream.IOStream, dest io.Writer, database st
return nil return nil
} }
// SQLShell executes a mysql shell inside the SQLStack. // OpenShell executes a mysql shell command
func (dis *Distillery) SQLShell(io stream.IOStream, argv ...string) (int, error) { func (sql SQLComponent) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return dis.SQLStack().Exec(io, "sql", "mysql", argv...) return sql.Stack().Exec(io, "sql", "mysql", argv...)
} }
const waitSQLInterval = 1 * time.Second // WaitShell waits for the sql database to be reachable via a docker-compose shell
func (sql SQLComponent) WaitShell() error {
// SQLWaitForShell waits for the sql database to be reachable via a docker-compose shell
func (dis *Distillery) SQLWaitForShell() error {
n := stream.FromNil() n := stream.FromNil()
return wait.Wait(func() bool { return wait.Wait(func() bool {
code, err := dis.SQLShell(n, "-e", "show databases;") code, err := sql.OpenShell(n, "-e", "show databases;")
return err == nil && code == 0 return err == nil && code == 0
}, waitSQLInterval, dis.Context()) }, sql.PollInterval, sql.dis.Context())
} }
// SQLWaitForConnection waits for the sql connection to be alive // Wait waits for a connection to the bookkeeping table to suceed
func (dis *Distillery) SQLWaitForConnection() error { func (sql SQLComponent) Wait() error {
return wait.Wait(func() bool { return wait.Wait(func() bool {
_, err := dis.sqlBkTable(true) _, err := sql.OpenBookkeeping(true)
return err == nil return err == nil
}, waitSQLInterval, dis.Context()) }, sql.PollInterval, sql.dis.Context())
} }
var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name") var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
func (dis *Distillery) sqlRaw(query string, args ...interface{}) bool { func (sql SQLComponent) Query(query string, args ...interface{}) bool {
sql := sqle.Format(query, args...) raw := sqle.Format(query, args...)
code, err := dis.SQLShell(stream.FromNil(), "-e", sql) code, err := sql.OpenShell(stream.FromNil(), "-e", raw)
return err == nil && code == 0 return err == nil && code == 0
} }
// SQLProvision provisions a new sql database and user // SQLProvision provisions a new sql database and user
func (dis *Distillery) SQLProvision(name, user, password string) error { func (sql SQLComponent) Provision(name, user, password string) error {
// wait for the database // wait for the database
if err := dis.SQLWaitForShell(); err != nil { if err := sql.WaitShell(); err != nil {
return err return err
} }
@ -138,7 +156,7 @@ func (dis *Distillery) SQLProvision(name, user, password string) error {
} }
// create the database and user! // create the database and user!
if !dis.sqlRaw("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) { if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
return errors.New("SQLProvision: Failed to create user") return errors.New("SQLProvision: Failed to create user")
} }
@ -149,8 +167,8 @@ func (dis *Distillery) SQLProvision(name, user, password string) error {
var errSQLPurgeUser = errors.New("unable to delete user") var errSQLPurgeUser = errors.New("unable to delete user")
// SQLPurgeUser deletes the specified user from the database // SQLPurgeUser deletes the specified user from the database
func (dis *Distillery) SQLPurgeUser(user string) error { func (sql SQLComponent) PurgeUser(user string) error {
if !dis.sqlRaw("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) { if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
return errSQLPurgeUser return errSQLPurgeUser
} }
@ -160,11 +178,11 @@ func (dis *Distillery) SQLPurgeUser(user string) error {
var errSQLPurgeDB = errors.New("unable to drop database") var errSQLPurgeDB = errors.New("unable to drop database")
// SQLPurgeDatabase deletes the specified db from the database // SQLPurgeDatabase deletes the specified db from the database
func (dis *Distillery) SQLPurgeDatabase(db string) error { func (sql SQLComponent) PurgeDatabase(db string) error {
if !sqle.IsSafeDatabaseName(db) { if !sqle.IsSafeDatabaseName(db) {
return errSQLPurgeDB return errSQLPurgeDB
} }
if !dis.sqlRaw("DROP DATABASE IF EXISTS `" + db + "`") { if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") {
return errSQLPurgeDB return errSQLPurgeDB
} }
return nil return nil
@ -174,18 +192,18 @@ var errSQLUnableToCreateUser = errors.New("unable to create administrative user"
var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe name") var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe name")
var errSQLUnableToCreate = errors.New("unable to create bookkeeping database") var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
// SQLBootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date // Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
func (dis *Distillery) SQLBootstrap(io stream.IOStream) error { func (sql SQLComponent) Bootstrap(io stream.IOStream) error {
if err := dis.SQLWaitForShell(); err != nil { if err := sql.WaitShell(); err != nil {
return err return err
} }
// create the admin user // create the admin user
logging.LogMessage(io, "Creating administrative user") logging.LogMessage(io, "Creating administrative user")
{ {
username := dis.Config.MysqlAdminUser username := sql.dis.Config.MysqlAdminUser
password := dis.Config.MysqlAdminPassword password := sql.dis.Config.MysqlAdminPassword
if !dis.sqlRaw("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) { if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
return errSQLUnableToCreateUser return errSQLUnableToCreateUser
} }
} }
@ -193,23 +211,23 @@ func (dis *Distillery) SQLBootstrap(io stream.IOStream) error {
// create the admin user // create the admin user
logging.LogMessage(io, "Creating sql database") logging.LogMessage(io, "Creating sql database")
{ {
if !sqle.IsSafeDatabaseName(dis.Config.DistilleryBookkeepingDatabase) { if !sqle.IsSafeDatabaseName(sql.dis.Config.DistilleryBookkeepingDatabase) {
return errSQLUnsafeDatabaseName return errSQLUnsafeDatabaseName
} }
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", dis.Config.DistilleryBookkeepingDatabase) createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.dis.Config.DistilleryBookkeepingDatabase)
if !dis.sqlRaw(createDBSQL) { if !sql.Query(createDBSQL) {
return errSQLUnableToCreate return errSQLUnableToCreate
} }
} }
// wait for the database to come up // wait for the database to come up
logging.LogMessage(io, "Waiting for database update to be complete") logging.LogMessage(io, "Waiting for database update to be complete")
dis.SQLWaitForConnection() sql.Wait()
// open the database // open the database
logging.LogMessage(io, "Migrating bookkeeping table") logging.LogMessage(io, "Migrating bookkeeping table")
{ {
db, err := dis.sqlBkTable(false) db, err := sql.OpenBookkeeping(false)
if err != nil { if err != nil {
return fmt.Errorf("unable to access bookkeeping table: %s", err) return fmt.Errorf("unable to access bookkeeping table: %s", err)
} }

23
env/stack_ssh.go vendored
View file

@ -2,11 +2,24 @@ package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack" import "github.com/FAU-CDI/wisski-distillery/internal/stack"
func (dis *Distillery) SSHStack() stack.Installable { // SSHComponent represents the 'ssh' layer belonging to a distillery
// TODO: Ensure that .env is copied if needed type SSHComponent struct {
return dis.asCoreStack("ssh", stack.Installable{}) dis *Distillery
} }
func (dis *Distillery) SSHStackPath() string { // SSH returns the SSHComponent belonging to this distillery
return dis.SSHStack().Dir func (dis *Distillery) SSH() SSHComponent {
return SSHComponent{dis: dis}
}
func (SSHComponent) Name() string {
return "ssh"
}
func (ssh SSHComponent) Stack() stack.Installable {
return ssh.dis.makeComponentStack(ssh, stack.Installable{})
}
func (ssh SSHComponent) Path() string {
return ssh.Stack().Dir
} }

View file

@ -20,8 +20,31 @@ import (
"github.com/tkw1536/goprogram/stream" "github.com/tkw1536/goprogram/stream"
) )
func (dis *Distillery) TriplestoreStack() stack.Installable { // TriplestoreComponent represents the triplestore belonging to a distillery
return dis.asCoreStack("triplestore", stack.Installable{ type TriplestoreComponent struct {
BaseURL string // the base url of the api
PollInterval time.Duration // duration to wait during wait!
dis *Distillery
}
// Triplestore returns the TriplestoreComponent belonging to this distillery
func (dis *Distillery) Triplestore() TriplestoreComponent {
return TriplestoreComponent{
BaseURL: "http://127.0.0.1:7200",
PollInterval: time.Second,
dis: dis,
}
}
func (TriplestoreComponent) Name() string {
return "triplestore"
}
// Stack returns the installable Triplestore stack
func (ts TriplestoreComponent) Stack() stack.Installable {
return ts.dis.makeComponentStack(ts, stack.Installable{
CopyContextFiles: []string{"graphdb.zip"}, CopyContextFiles: []string{"graphdb.zip"},
MakeDirsPerm: fs.ModeDir | fs.ModePerm, MakeDirsPerm: fs.ModeDir | fs.ModePerm,
@ -33,8 +56,8 @@ func (dis *Distillery) TriplestoreStack() stack.Installable {
}) })
} }
func (dis *Distillery) TriplestoreStackPath() string { func (ts TriplestoreComponent) Path() string {
return dis.TriplestoreStack().Dir return ts.Stack().Dir
} }
type TriplestoreUserPayload struct { type TriplestoreUserPayload struct {
@ -50,14 +73,11 @@ type TriplestoreUserAppSettings struct {
ExecuteCount bool `json:"EXECUTE_COUNT"` ExecuteCount bool `json:"EXECUTE_COUNT"`
} }
const triplestoreBaseURL = "http://127.0.0.1:7200" // OpenRaw makes an http request to the triplestore api.
const waitTSInterval = 1 * time.Second
// triplestoreCall makes a request to the triplestore.
// //
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name. // When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
// When bodyName is empty, simply marshal body as application/json // When bodyName is empty, simply marshal body as application/json
func (dis *Distillery) triplestoreRequest(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) { func (ts TriplestoreComponent) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
var reader io.Reader var reader io.Reader
var contentType string var contentType string
@ -87,7 +107,7 @@ func (dis *Distillery) triplestoreRequest(method, url string, body interface{},
} }
// create the request object // create the request object
req, err := http.NewRequest(method, triplestoreBaseURL+url, reader) req, err := http.NewRequest(method, ts.BaseURL+url, reader)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -99,21 +119,23 @@ func (dis *Distillery) triplestoreRequest(method, url string, body interface{},
if contentType != "" { if contentType != "" {
req.Header.Set("Content-Type", contentType) req.Header.Set("Content-Type", contentType)
} }
req.SetBasicAuth(dis.Config.TriplestoreAdminUser, dis.Config.TriplestoreAdminPassword) req.SetBasicAuth(ts.dis.Config.TriplestoreAdminUser, ts.dis.Config.TriplestoreAdminPassword)
// and send it // and send it
return http.DefaultClient.Do(req) return http.DefaultClient.Do(req)
} }
func (dis *Distillery) TriplestoreWaitForConnection() error { // Wait waits for the connection to the Triplestore to succeed.
// This is achieved using a polling strategy.
func (ts TriplestoreComponent) Wait() error {
return wait.Wait(func() bool { return wait.Wait(func() bool {
res, err := dis.triplestoreRequest("GET", "/rest/repositories", nil, "", "") res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
if err != nil { if err != nil {
return false return false
} }
defer res.Body.Close() defer res.Body.Close()
return true return true
}, waitTSInterval, dis.Context()) }, ts.PollInterval, ts.dis.Context())
} }
var errTripleStoreFailedRepository = exit.Error{ var errTripleStoreFailedRepository = exit.Error{
@ -121,8 +143,8 @@ var errTripleStoreFailedRepository = exit.Error{
ExitCode: exit.ExitGeneric, ExitCode: exit.ExitGeneric,
} }
func (dis *Distillery) TriplestoreProvision(name, domain, user, password string) error { func (ts TriplestoreComponent) Provision(name, domain, user, password string) error {
if err := dis.TriplestoreWaitForConnection(); err != nil { if err := ts.Wait(); err != nil {
return err return err
} }
@ -137,7 +159,7 @@ func (dis *Distillery) TriplestoreProvision(name, domain, user, password string)
// do the create! // do the create!
{ {
res, err := dis.triplestoreRequest("POST", "/rest/repositories", createRepo, "config", "") res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo, "config", "")
if err != nil { if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err) return errTripleStoreFailedRepository.WithMessageF(err)
} }
@ -149,7 +171,7 @@ func (dis *Distillery) TriplestoreProvision(name, domain, user, password string)
// create the user and grant them access // create the user and grant them access
{ {
res, err := dis.triplestoreRequest("POST", "/rest/security/users/"+user, TriplestoreUserPayload{ res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
Password: password, Password: password,
AppSettings: TriplestoreUserAppSettings{ AppSettings: TriplestoreUserAppSettings{
DefaultInference: true, DefaultInference: true,
@ -177,8 +199,8 @@ func (dis *Distillery) TriplestoreProvision(name, domain, user, password string)
} }
// TriplestorePurgeUser deletes the specified user from the triplestore // TriplestorePurgeUser deletes the specified user from the triplestore
func (dis *Distillery) TriplestorePurgeUser(user string) error { func (ts TriplestoreComponent) PurgeUser(user string) error {
res, err := dis.triplestoreRequest("DELETE", "/rest/security/users/"+user, nil, "", "") res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
if err != nil { if err != nil {
return err return err
} }
@ -189,8 +211,8 @@ func (dis *Distillery) TriplestorePurgeUser(user string) error {
} }
// TriplestorePurgeRepo deletes the specified repo from the triplestore // TriplestorePurgeRepo deletes the specified repo from the triplestore
func (dis *Distillery) TriplestorePurgeRepo(repo string) error { func (ts TriplestoreComponent) PurgeRepo(repo string) error {
res, err := dis.triplestoreRequest("DELETE", "/rest/repositories/"+repo, nil, "", "") res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
if err != nil { if err != nil {
return err return err
} }
@ -203,8 +225,8 @@ func (dis *Distillery) TriplestorePurgeRepo(repo string) error {
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code") var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
// TriplestoreBackup backs up the repository named repo into the writer dst. // TriplestoreBackup backs up the repository named repo into the writer dst.
func (dis *Distillery) TriplestoreBackup(dst io.Writer, repo string) (int64, error) { func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error) {
res, err := dis.triplestoreRequest("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads") res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
if err != nil { if err != nil {
return 0, err return 0, err
} }
@ -217,16 +239,16 @@ func (dis *Distillery) TriplestoreBackup(dst io.Writer, repo string) (int64, err
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK") var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error { func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {
logging.LogMessage(io, "Waiting for Triplestore") logging.LogMessage(io, "Waiting for Triplestore")
if err := dis.TriplestoreWaitForConnection(); err != nil { if err := ts.Wait(); err != nil {
return err return err
} }
logging.LogMessage(io, "Resetting admin user password") logging.LogMessage(io, "Resetting admin user password")
{ {
res, err := dis.triplestoreRequest("PUT", "/rest/security/users/"+dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{ res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{
Password: dis.Config.TriplestoreAdminPassword, Password: ts.dis.Config.TriplestoreAdminPassword,
AppSettings: TriplestoreUserAppSettings{ AppSettings: TriplestoreUserAppSettings{
DefaultInference: true, DefaultInference: true,
DefaultVisGraphSchema: true, DefaultVisGraphSchema: true,
@ -257,7 +279,7 @@ func (dis *Distillery) TriplestoreBootstrap(io stream.IOStream) error {
logging.LogMessage(io, "Enabling Triplestore security") logging.LogMessage(io, "Enabling Triplestore security")
{ {
res, err := dis.triplestoreRequest("POST", "/rest/security", true, "", "") res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
if err != nil { if err != nil {
return fmt.Errorf("failed to enable triplestore security: %s", err) return fmt.Errorf("failed to enable triplestore security: %s", err)
} }

24
env/stack_web.go vendored
View file

@ -2,14 +2,28 @@ package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack" import "github.com/FAU-CDI/wisski-distillery/internal/stack"
func (dis *Distillery) WebStack() stack.Installable { // WebComponent represents the 'web' layer belonging to a distillery
return dis.asCoreStack("web", stack.Installable{ type WebComponent struct {
dis *Distillery
}
// Web returns the WebComponent belonging to this distillery
func (dis *Distillery) Web() WebComponent {
return WebComponent{dis: dis}
}
func (WebComponent) Name() string {
return "web"
}
func (web WebComponent) Stack() stack.Installable {
return web.dis.makeComponentStack(web, stack.Installable{
EnvFileContext: map[string]string{ EnvFileContext: map[string]string{
"DEFAULT_HOST": dis.Config.DefaultDomain, "DEFAULT_HOST": web.dis.Config.DefaultDomain,
}, },
}) })
} }
func (dis *Distillery) WebStackPath() string { func (web WebComponent) Path() string {
return dis.WebStack().Dir return web.Stack().Dir
} }