internal/component => internal/dis/component

internal/dis/component/sql/backup.go

@@ -0,0 +1,30 @@
+package sql
+
+import (
+	"errors"
+	"io"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
+)
+
+var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
+
+func (*SQL) BackupName() string {
+	return "sql.sql"
+}
+
+// Backup makes a backup of all SQL databases into the path dest.
+func (sql *SQL) Backup(context component.StagingContext) error {
+	return context.AddFile("", func(file io.Writer) error {
+		io := context.IO().Streams(file, nil, nil, 0).NonInteractive()
+		code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--all-databases")
+		if err != nil {
+			return err
+		}
+		if code != 0 {
+			return errSQLBackup
+		}
+		return nil
+	})
+
+}

internal/dis/component/sql/connect.go

@@ -0,0 +1,140 @@
+package sql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	mysqldriver "github.com/go-sql-driver/mysql"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/timex"
+)
+
+//
+// ========== low-level connection ==========
+//
+
+// Exec executes a database-independent database query.
+func (sql *SQL) Exec(query string, args ...interface{}) error {
+	// connect to the server
+	conn, err := sql.connect("")
+	if err != nil {
+		return err
+	}
+
+	// do the query!
+	{
+		_, err := conn.Exec(query, args...)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+}
+
+// WaitExec waits for the query interface to be able to connect to the database
+func (sql *SQL) WaitExec() error {
+	return timex.TickUntilFunc(func(time.Time) bool {
+		err := sql.Exec("select 1;")
+		return err == nil
+	}, sql.PollContext, sql.PollInterval)
+}
+
+//
+// ========== connection via gorm ==========
+//
+
+// QueryTable returns a gorm.DB to connect to the provided distillery database table
+func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
+	conn, err := sql.connect(sql.Config.DistilleryDatabase)
+	if err != nil {
+		return nil, err
+	}
+
+	// gorm configuration
+	config := &gorm.Config{}
+	if silent {
+		config.Logger = logger.Default.LogMode(logger.Silent)
+	}
+
+	// mysql connection
+	cfg := mysql.Config{
+		Conn: conn,
+
+		DefaultStringSize: 256,
+	}
+
+	// open the gorm connection!
+	db, err := gorm.Open(mysql.New(cfg), config)
+	if err != nil {
+		return nil, err
+	}
+
+	// set the table
+	db = db.Table(table)
+
+	// check that nothing went wrong
+	if db.Error != nil {
+		return nil, db.Error
+	}
+	return db, nil
+}
+
+// WaitQueryTable waits for a connection to succeed via QueryTable
+func (sql *SQL) WaitQueryTable() error {
+	// TODO: Establish a convention on when to wait for this!
+	return timex.TickUntilFunc(func(time.Time) bool {
+		_, err := sql.QueryTable(true, models.InstanceTable)
+		return err == nil
+	}, sql.PollContext, sql.PollInterval)
+}
+
+//
+// ========== low-level database connection ==========
+//
+
+func (ssql *SQL) connect(database string) (*sql.DB, error) {
+	conn, err := sql.Open("mysql", ssql.dsn(database))
+	if err != nil {
+		return nil, err
+	}
+
+	conn.SetMaxIdleConns(0)
+
+	return conn, nil
+}
+
+// dsn returns a dsn fof connecting to the database
+func (sql *SQL) dsn(database string) string {
+	user := sql.Config.MysqlAdminUser
+	pass := sql.Config.MysqlAdminPassword
+	network := sql.network()
+	server := sql.ServerURL
+
+	return fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8&parseTime=True&loc=Local", user, pass, network, server, database)
+}
+
+var proxyNameCounter uint64
+
+// network returns the network to use to connect to the database
+func (sql *SQL) network() string {
+	return sql.lazyNetwork.Get(func() (name string) {
+		network := "tcp"
+
+		// register a new DialContext function to use the environment.
+		// this seems like a bit of a hack, but it works for now.
+		name = fmt.Sprintf("sql-network-%d", atomic.AddUint64(&proxyNameCounter, 1))
+		mysqldriver.RegisterDialContext(name, func(ctx context.Context, addr string) (net.Conn, error) {
+			return sql.Still.Environment.DialContext(ctx, network, addr)
+		})
+		return
+	})
+}

internal/dis/component/sql/provision.go

@@ -0,0 +1,120 @@
+package sql
+
+import (
+	"errors"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/errorx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
+)
+
+var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters")
+var errProvisionInvalidGrant = errors.New("Provision: Grant failed")
+
+// Provision provisions sql-specific resource for the given instance
+func (sql *SQL) Provision(instance models.Instance, domain string) error {
+	return sql.CreateDatabase(instance.SqlDatabase, instance.SqlUsername, instance.SqlPassword)
+}
+
+// Purge purges sql-specific resources for the given instance
+func (sql *SQL) Purge(instance models.Instance, domain string) error {
+	return errorx.First(
+		sql.PurgeDatabase(instance.SqlDatabase),
+		sql.PurgeUser(instance.SqlUsername),
+	)
+}
+
+// CreateDatabase creates a new database with the given name.
+// It then generates a new user, with the name 'user' and the password 'password', that is then granted access to this database.
+//
+// Provision internally waits for the database to become available.
+func (sql *SQL) CreateDatabase(name, user, password string) error {
+
+	// NOTE(twiesing): We shouldn't use string concat to build sql queries.
+	// But the driver doesn't support using query params for this particular query.
+	// Apparently it's a "feature", see https://github.com/go-sql-driver/mysql/issues/398#issuecomment-169951763.
+
+	// quick and dirty check to make sure that all the names won't sql inject.
+	if !sqle.IsSafeDatabaseLiteral(name) || !sqle.IsSafeDatabaseSingleQuote(user) || !sqle.IsSafeDatabaseSingleQuote(password) {
+		return errProvisionInvalidDatabaseParams
+	}
+
+	// We use the sql shell here, because not only can we not use query params, but the driver outright rejects queries.
+	// Queries of the form "CREATE USER 'test'@'%' IDENTIFIED BY 'test'; FLUSH PRIVILEGES;" return error 1064 when using driver, but are fine with the shell.
+	// This should be fixed eventually, but I have no idea how.
+
+	if err := sql.unsafeWaitShell(); err != nil {
+		return err
+	}
+
+	query := "CREATE DATABASE `" + name + "`;" +
+		"CREATE USER '" + user + "'@'%' IDENTIFIED BY '" + password + "';" +
+		"GRANT ALL PRIVILEGES ON `" + name + "`.* TO `" + user + "`@`%`; FLUSH PRIVILEGES;"
+	if !sql.unsafeQueryShell(query) {
+		return errProvisionInvalidGrant
+	}
+
+	return nil
+}
+
+var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed")
+
+// CreateSuperuser createsa new user, with the name 'user' and the password 'password'.
+// It then grants this user superuser status in the database.
+//
+// CreateSuperuser internally waits for the database to become available.
+func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error {
+	// NOTE(twiesing): This function unsafely uses the shell directly to create a superuser.
+	// This is for two reasons:
+	// (1) this is used during bootstraping
+	// (2) The underlying driver doesn't support "GRANT ALL PRIVILEGES"
+	// See also [sql.Provision].
+
+	if !sqle.IsSafeDatabaseSingleQuote(user) || !sqle.IsSafeDatabaseSingleQuote(password) {
+		return errProvisionInvalidDatabaseParams
+	}
+
+	if err := sql.unsafeWaitShell(); err != nil {
+		return err
+	}
+
+	var IfNotExists string
+	if allowExisting {
+		IfNotExists = "IF NOT EXISTS"
+	}
+
+	query := "CREATE USER " + IfNotExists + " '" + user + "'@'%' IDENTIFIED BY '" + password + "';" +
+		"GRANT ALL PRIVILEGES ON *.* TO '" + user + "'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;"
+	if !sql.unsafeQueryShell(query) {
+		return errCreateSuperuserGrant
+	}
+
+	return nil
+}
+
+var errPurgeUser = errors.New("PurgeUser: Failed to drop user")
+
+// SQLPurgeUser deletes the specified user from the database
+func (sql *SQL) PurgeUser(user string) error {
+	if !sqle.IsSafeDatabaseSingleQuote(user) {
+		return errPurgeUser
+	}
+
+	query := "DROP USER IF EXISTS '" + user + "'@'%';" +
+		"FLUSH PRIVILEGES;"
+	if !sql.unsafeQueryShell(query) {
+		return errPurgeUser
+	}
+
+	return nil
+}
+
+var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name")
+
+// SQLPurgeDatabase deletes the specified db from the database
+func (sql *SQL) PurgeDatabase(db string) error {
+	if !sqle.IsSafeDatabaseLiteral(db) {
+		return errSQLPurgeDB
+	}
+	return sql.Exec("DROP DATABASE IF EXISTS `" + db + "`")
+}

internal/dis/component/sql/snapshot.go

@@ -0,0 +1,35 @@
+package sql
+
+import (
+	"io"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+func (*SQL) SnapshotNeedsRunning() bool { return false }
+
+func (*SQL) SnapshotName() string { return "sql" }
+
+func (sql *SQL) Snapshot(wisski models.Instance, context component.StagingContext) error {
+	return context.AddDirectory(".", func() error {
+		return context.AddFile(wisski.SqlDatabase+".sql", func(file io.Writer) error {
+			return sql.SnapshotDB(context.IO(), file, wisski.SqlDatabase)
+		})
+	})
+}
+
+// SnapshotDB makes a backup of the sql database into dest.
+func (sql *SQL) SnapshotDB(io stream.IOStream, dest io.Writer, database string) error {
+	io = io.Streams(dest, nil, nil, 0).NonInteractive()
+
+	code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database)
+	if err != nil {
+		return err
+	}
+	if code != 0 {
+		return errSQLBackup
+	}
+	return nil
+}

internal/dis/component/sql/sql.env

@@ -0,0 +1,2 @@
+DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME}
+HTTPS_ENABLED=${HTTPS_ENABLED}

internal/dis/component/sql/sql.go

@@ -0,0 +1,53 @@
+package sql
+
+import (
+	"context"
+	"embed"
+	"path/filepath"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
+	"github.com/FAU-CDI/wisski-distillery/pkg/environment"
+	"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
+)
+
+type SQL struct {
+	component.Base
+
+	ServerURL string // upstream server url
+
+	PollContext  context.Context // context to abort polling with
+	PollInterval time.Duration   // duration to wait for during wait
+
+	lazyNetwork lazy.Lazy[string]
+}
+
+func (sql *SQL) Path() string {
+	return filepath.Join(sql.Still.Config.DeployRoot, "core", "sql")
+}
+
+func (*SQL) Context(parent component.InstallationContext) component.InstallationContext {
+	return parent
+}
+
+//go:embed all:sql
+//go:embed sql.env
+var resources embed.FS
+
+func (sql *SQL) Stack(env environment.Environment) component.StackWithResources {
+	return component.MakeStack(sql, env, component.StackWithResources{
+		Resources:   resources,
+		ContextPath: "sql",
+
+		EnvPath: "sql.env",
+		EnvContext: map[string]string{
+			"DOCKER_NETWORK_NAME": sql.Config.DockerNetworkName,
+			"HTTPS_ENABLED":       sql.Config.HTTPSEnabledEnv(),
+		},
+
+		MakeDirsPerm: environment.DefaultDirPerm,
+		MakeDirs: []string{
+			"data",
+		},
+	})
+}

internal/dis/component/sql/sql/docker-compose.yml

@@ -0,0 +1,39 @@
+version: "3.7"
+
+services:
+    sql:
+        image: mariadb
+        volumes:
+            - "./data/:/var/lib/mysql"
+        ports:
+            - 127.0.0.1:3306:3306
+        labels:
+            - "eu.wiss-ki.barrel.distillery=${DOCKER_NETWORK_NAME}"
+        environment:
+            # This combination of environment variables will configure a passwordless root user
+            # that can only connect to the container from 'localhost'.
+            # This means we can only connect using 'docker-compose exec sql mysql -C '...' '. 
+            - "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
+            - "MYSQL_ROOT_HOST=localhost"
+        restart: always
+    phpmyadmin:
+        image: phpmyadmin/phpmyadmin
+        environment:
+            - "PMA_HOST=sql"
+            - "HIDE_PHP_VERSION=true"
+            - "UPLOAD_LIMIT=100M"
+        # phpmyadmin running on localhost:8080 so that we can easily access the system graphically. 
+        # By default no admin account is created, so initial shell access to make one is needed. 
+        ports:
+            - 127.0.0.1:8080:80
+        labels:
+            - "eu.wiss-ki.barrel.distillery=${DOCKER_NETWORK_NAME}"
+        depends_on:
+            - sql
+        restart: always
+
+
+networks:
+    default:
+        name: ${DOCKER_NETWORK_NAME}
+        external: true

internal/dis/component/sql/update.go

@@ -0,0 +1,121 @@
+package sql
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/models"
+	"github.com/FAU-CDI/wisski-distillery/pkg/logging"
+	"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
+	"github.com/FAU-CDI/wisski-distillery/pkg/timex"
+	"github.com/tkw1536/goprogram/exit"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Shell runs a mysql shell with the provided databases.
+//
+// NOTE(twiesing): This command should not be used to connect to the database or execute queries except in known situations.
+func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) {
+	return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
+}
+
+// unsafeWaitShell waits for a connection via the database shell to succeed
+func (sql *SQL) unsafeWaitShell() error {
+	n := stream.FromNil()
+	return timex.TickUntilFunc(func(time.Time) bool {
+		code, err := sql.Shell(n, "-e", "select 1;")
+		return err == nil && code == 0
+	}, sql.PollContext, sql.PollInterval)
+}
+
+// unsafeQuery shell executes a raw database query.
+func (sql *SQL) unsafeQueryShell(query string) bool {
+	code, err := sql.Shell(stream.FromNil(), "-e", query)
+	return err == nil && code == 0
+}
+
+var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
+var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name")
+var errSQLUnableToMigrate = exit.Error{
+	Message:  "unable to migrate %s table: %s",
+	ExitCode: exit.ExitGeneric,
+}
+
+// Update initializes or updates the SQL database.
+func (sql *SQL) Update(io stream.IOStream) error {
+
+	// unsafely create the admin user!
+	{
+		if err := sql.unsafeWaitShell(); err != nil {
+			return err
+		}
+		logging.LogMessage(io, "Creating administrative user")
+		{
+			username := sql.Config.MysqlAdminUser
+			password := sql.Config.MysqlAdminPassword
+			if err := sql.CreateSuperuser(username, password, true); err != nil {
+				return errSQLUnableToCreateUser
+			}
+		}
+	}
+
+	// create the admin user
+	logging.LogMessage(io, "Creating sql database")
+	{
+		if !sqle.IsSafeDatabaseLiteral(sql.Config.DistilleryDatabase) {
+			return errSQLUnsafeDatabaseName
+		}
+		createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase)
+		if err := sql.Exec(createDBSQL); err != nil {
+			return err
+		}
+	}
+
+	// wait for the database to come up
+	logging.LogMessage(io, "Waiting for database update to be complete")
+	sql.WaitQueryTable()
+
+	tables := []struct {
+		name  string
+		model any
+		table string
+	}{
+		{
+			"instance",
+			&models.Instance{},
+			models.InstanceTable,
+		},
+		{
+			"metadata",
+			&models.Metadatum{},
+			models.MetadataTable,
+		},
+		{
+			"snapshot",
+			&models.Export{},
+			models.ExportTable,
+		},
+		{
+			"lock",
+			&models.Lock{},
+			models.LockTable,
+		},
+	}
+
+	// migrate all of the tables!
+	return logging.LogOperation(func() error {
+		for _, table := range tables {
+			logging.LogMessage(io, "migrating %q table", table.name)
+			db, err := sql.QueryTable(false, table.table)
+			if err != nil {
+				return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
+			}
+
+			if err := db.AutoMigrate(table.model); err != nil {
+				return errSQLUnableToMigrate.WithMessageF(table.name, err)
+			}
+		}
+		return nil
+	}, io, "migrating database tables")
+}