From f7b8804c6189abf0212e33da3e082d459895e9e1 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Mon, 19 Sep 2022 13:03:59 +0200 Subject: [PATCH] sql: Begin cleanup of code --- cmd/backup.go | 1 - cmd/make_mysql_account.go | 2 +- cmd/mysql.go | 2 +- internal/component/sql/bootstrap.go | 63 ++++++++++++ internal/component/sql/database.go | 150 ++++------------------------ internal/component/sql/provision.go | 52 ++++++++++ internal/component/sql/snapshot.go | 21 ++++ internal/component/sql/sql.go | 2 +- 8 files changed, 158 insertions(+), 135 deletions(-) create mode 100644 internal/component/sql/bootstrap.go create mode 100644 internal/component/sql/provision.go create mode 100644 internal/component/sql/snapshot.go diff --git a/cmd/backup.go b/cmd/backup.go index 0c9f668..2aac230 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -101,7 +101,6 @@ func (bk backupC) Run(context wisski_distillery.Context) error { } // and write everything into it! - // TODO: Should we move the open call to here? var count int64 if err := logging.LogOperation(func() error { context.IOStream.Println(archivePath) diff --git a/cmd/make_mysql_account.go b/cmd/make_mysql_account.go index d0ff5b5..74fee7b 100644 --- a/cmd/make_mysql_account.go +++ b/cmd/make_mysql_account.go @@ -55,7 +55,7 @@ func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error { if err != nil { return err } - code, err := context.Environment.SQL().OpenShell(context.IOStream, "-e", query) + code, err := context.Environment.SQL().Shell(context.IOStream, "-e", query) if err != nil { return err } diff --git a/cmd/mysql.go b/cmd/mysql.go index dad794f..b840db0 100644 --- a/cmd/mysql.go +++ b/cmd/mysql.go @@ -32,7 +32,7 @@ func (mysql) Description() wisski_distillery.Description { } func (ms mysql) Run(context wisski_distillery.Context) error { - code, err := context.Environment.SQL().OpenShell(context.IOStream, ms.Positionals.Args...) + code, err := context.Environment.SQL().Shell(context.IOStream, ms.Positionals.Args...) if err != nil { return err } diff --git a/internal/component/sql/bootstrap.go b/internal/component/sql/bootstrap.go new file mode 100644 index 0000000..ef18c6b --- /dev/null +++ b/internal/component/sql/bootstrap.go @@ -0,0 +1,63 @@ +package sql + +import ( + "errors" + "fmt" + + "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" + "github.com/FAU-CDI/wisski-distillery/pkg/logging" + "github.com/FAU-CDI/wisski-distillery/pkg/sqle" + "github.com/tkw1536/goprogram/stream" +) + +var errSQLUnableToCreateUser = errors.New("unable to create administrative user") +var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name") +var errSQLUnableToCreate = errors.New("unable to create bookkeeping database") + +// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date +func (sql *SQL) Bootstrap(io stream.IOStream) error { + if err := sql.WaitShell(); err != nil { + return err + } + + // create the admin user + logging.LogMessage(io, "Creating administrative user") + { + username := sql.Config.MysqlAdminUser + password := sql.Config.MysqlAdminPassword + if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) { + return errSQLUnableToCreateUser + } + } + + // create the admin user + logging.LogMessage(io, "Creating sql database") + { + if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) { + return errSQLUnsafeDatabaseName + } + createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase) + if !sql.Query(createDBSQL) { + return errSQLUnableToCreate + } + } + + // wait for the database to come up + logging.LogMessage(io, "Waiting for database update to be complete") + sql.Wait() + + // open the database + logging.LogMessage(io, "Migrating bookkeeping table") + { + db, err := sql.OpenBookkeeping(false) + if err != nil { + return fmt.Errorf("unable to access bookkeeping table: %s", err) + } + + if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil { + return fmt.Errorf("unable to migrate bookkeeping table: %s", err) + } + } + + return nil +} diff --git a/internal/component/sql/database.go b/internal/component/sql/database.go index 46e56d0..fe5075f 100644 --- a/internal/component/sql/database.go +++ b/internal/component/sql/database.go @@ -4,14 +4,11 @@ import ( "context" "errors" "fmt" - "io" "net" "sync/atomic" mysqldriver "github.com/go-sql-driver/mysql" - "github.com/FAU-CDI/wisski-distillery/internal/bookkeeping" - "github.com/FAU-CDI/wisski-distillery/pkg/logging" "github.com/FAU-CDI/wisski-distillery/pkg/sqle" "github.com/FAU-CDI/wisski-distillery/pkg/wait" "github.com/tkw1536/goprogram/stream" @@ -22,25 +19,26 @@ import ( var proxyNameCounter uint64 -// registerDialingProxy registers a new custom network protocol with the underlying sql driver. -// The new protocol will call dialer with the provided network argument. -// The name of the new protocol is returned. -func registerDialingProxy(network string, dialer func(context context.Context, network, address string) (net.Conn, error)) (name string) { - name = fmt.Sprintf("sql-proxy-%d", atomic.AddUint64(&proxyNameCounter, 1)) - mysqldriver.RegisterDialContext(name, func(ctx context.Context, addr string) (net.Conn, error) { - return dialer(ctx, network, addr) +// network returns the network to use to connect to the database +func (sql *SQL) network() string { + return sql.lazyNetwork.Get(func() (name string) { + network := "tcp" + + // register a new DialContext function to use the environment. + // this seems like a bit of a hack, but it works for now. + name = fmt.Sprintf("sql-network-%d", atomic.AddUint64(&proxyNameCounter, 1)) + mysqldriver.RegisterDialContext(name, func(ctx context.Context, addr string) (net.Conn, error) { + return sql.Core.Environment.DialContext(ctx, network, addr) + }) + return }) - return } // sqlOpen opens a new sql connection to the provided database using the administrative credentials func (sql *SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) { - network := sql.sqlNetwork.Get(func() string { - return registerDialingProxy("tcp", sql.Core.Environment.DialContext) - }) cfg := mysql.Config{ DriverName: "mysql", - DSN: fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, network, sql.ServerURL, database), + DSN: fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.network(), sql.ServerURL, database), DefaultStringSize: 256, } @@ -81,30 +79,16 @@ func (sql *SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) { return table, nil } -// Snapshot makes a backup of the sql database into dest. -func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error { - io = io.Streams(dest, nil, nil, 0).NonInteractive() - - code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database) - if err != nil { - return err - } - if code != 0 { - return errSQLBackup - } - return nil -} - -// OpenShell executes a mysql shell command -func (sql *SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) { +// Shell runs a mysql shell command. +func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) { return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...) } -// WaitShell waits for the sql database to be reachable via a docker-compose shell +// WaitShell waits for the sql database to be reachable via shell func (sql *SQL) WaitShell() error { n := stream.FromNil() return wait.Wait(func() bool { - code, err := sql.OpenShell(n, "-e", "show databases;") + code, err := sql.Shell(n, "-e", "show databases;") return err == nil && code == 0 }, sql.PollInterval, sql.PollContext) } @@ -119,105 +103,9 @@ func (sql *SQL) Wait() error { var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name") +// Query performs a raw database query func (sql *SQL) Query(query string, args ...interface{}) bool { raw := sqle.Format(query, args...) - code, err := sql.OpenShell(stream.FromNil(), "-e", raw) + code, err := sql.Shell(stream.FromNil(), "-e", raw) return err == nil && code == 0 } - -// SQLProvision provisions a new sql database and user -func (sql *SQL) Provision(name, user, password string) error { - // wait for the database - if err := sql.WaitShell(); err != nil { - return err - } - - // it's not a safe database name! - if !sqle.IsSafeDatabaseName(name) { - return errInvalidDatabaseName - } - - // create the database and user! - if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) { - return errors.New("SQLProvision: Failed to create user") - } - - // and done! - return nil -} - -var errSQLPurgeUser = errors.New("unable to delete user") - -// SQLPurgeUser deletes the specified user from the database -func (sql *SQL) PurgeUser(user string) error { - if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) { - return errSQLPurgeUser - } - - return nil -} - -var errSQLPurgeDB = errors.New("unable to drop database") - -// SQLPurgeDatabase deletes the specified db from the database -func (sql *SQL) PurgeDatabase(db string) error { - if !sqle.IsSafeDatabaseName(db) { - return errSQLPurgeDB - } - if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") { - return errSQLPurgeDB - } - return nil -} - -var errSQLUnableToCreateUser = errors.New("unable to create administrative user") -var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name") -var errSQLUnableToCreate = errors.New("unable to create bookkeeping database") - -// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date -func (sql *SQL) Bootstrap(io stream.IOStream) error { - if err := sql.WaitShell(); err != nil { - return err - } - - // create the admin user - logging.LogMessage(io, "Creating administrative user") - { - username := sql.Config.MysqlAdminUser - password := sql.Config.MysqlAdminPassword - if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) { - return errSQLUnableToCreateUser - } - } - - // create the admin user - logging.LogMessage(io, "Creating sql database") - { - if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) { - return errSQLUnsafeDatabaseName - } - createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase) - if !sql.Query(createDBSQL) { - return errSQLUnableToCreate - } - } - - // wait for the database to come up - logging.LogMessage(io, "Waiting for database update to be complete") - sql.Wait() - - // open the database - logging.LogMessage(io, "Migrating bookkeeping table") - { - db, err := sql.OpenBookkeeping(false) - if err != nil { - return fmt.Errorf("unable to access bookkeeping table: %s", err) - } - - if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil { - return fmt.Errorf("unable to migrate bookkeeping table: %s", err) - } - } - - return nil -} diff --git a/internal/component/sql/provision.go b/internal/component/sql/provision.go new file mode 100644 index 0000000..e2ecf8a --- /dev/null +++ b/internal/component/sql/provision.go @@ -0,0 +1,52 @@ +package sql + +import ( + "errors" + + "github.com/FAU-CDI/wisski-distillery/pkg/sqle" +) + +// SQLProvision provisions a new sql database and user +func (sql *SQL) Provision(name, user, password string) error { + // wait for the database + if err := sql.WaitShell(); err != nil { + return err + } + + // it's not a safe database name! + if !sqle.IsSafeDatabaseName(name) { + return errInvalidDatabaseName + } + + // create the database and user! + if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) { + return errors.New("SQLProvision: Failed to create user") + } + + // and done! + return nil +} + +var errSQLPurgeUser = errors.New("unable to delete user") + +// SQLPurgeUser deletes the specified user from the database +func (sql *SQL) PurgeUser(user string) error { + if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) { + return errSQLPurgeUser + } + + return nil +} + +var errSQLPurgeDB = errors.New("unable to drop database") + +// SQLPurgeDatabase deletes the specified db from the database +func (sql *SQL) PurgeDatabase(db string) error { + if !sqle.IsSafeDatabaseName(db) { + return errSQLPurgeDB + } + if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") { + return errSQLPurgeDB + } + return nil +} diff --git a/internal/component/sql/snapshot.go b/internal/component/sql/snapshot.go new file mode 100644 index 0000000..3a73f83 --- /dev/null +++ b/internal/component/sql/snapshot.go @@ -0,0 +1,21 @@ +package sql + +import ( + "io" + + "github.com/tkw1536/goprogram/stream" +) + +// Snapshot makes a backup of the sql database into dest. +func (sql SQL) Snapshot(io stream.IOStream, dest io.Writer, database string) error { + io = io.Streams(dest, nil, nil, 0).NonInteractive() + + code, err := sql.Stack(sql.Environment).Exec(io, "sql", "mysqldump", "--databases", database) + if err != nil { + return err + } + if code != 0 { + return errSQLBackup + } + return nil +} diff --git a/internal/component/sql/sql.go b/internal/component/sql/sql.go index 6369cdc..cf1fdcf 100644 --- a/internal/component/sql/sql.go +++ b/internal/component/sql/sql.go @@ -18,7 +18,7 @@ type SQL struct { PollContext context.Context // context to abort polling with PollInterval time.Duration // duration to wait for during wait - sqlNetwork lazy.Lazy[string] + lazyNetwork lazy.Lazy[string] } func (SQL) Name() string {