Cleanup and document hacky sql interaction
This commit is contained in:
parent
881b538dff
commit
07409a01be
17 changed files with 284 additions and 204 deletions
|
|
@ -1,11 +1,8 @@
|
||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
|
|
||||||
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
|
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
|
||||||
"github.com/tkw1536/goprogram/exit"
|
"github.com/tkw1536/goprogram/exit"
|
||||||
"github.com/tkw1536/goprogram/parser"
|
"github.com/tkw1536/goprogram/parser"
|
||||||
)
|
)
|
||||||
|
|
@ -39,6 +36,8 @@ var errUnableToReadPassword = exit.Error{
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error {
|
func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error {
|
||||||
|
dis := context.Environment
|
||||||
|
|
||||||
context.Printf("Username>")
|
context.Printf("Username>")
|
||||||
username, err := context.ReadLine()
|
username, err := context.ReadLine()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -51,20 +50,9 @@ func (mma makeMysqlAccount) Run(context wisski_distillery.Context) error {
|
||||||
return errUnableToReadPassword.WithMessageF(err)
|
return errUnableToReadPassword.WithMessageF(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
query := sqle.Format("CREATE USER ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username)
|
if err := dis.SQL().CreateSuperuser(username, password, false); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
code, err := context.Environment.SQL().Shell(context.IOStream, "-e", query)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if code != 0 {
|
|
||||||
return exit.Error{
|
|
||||||
ExitCode: exit.ExitCode(uint8(code)),
|
|
||||||
Message: fmt.Sprintf("Exit code %d", code),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,8 +25,7 @@
|
||||||
<b>GraphDB User Prefix:</b> <code>{{.Config.GraphDBUserPrefix}}</code><br />
|
<b>GraphDB User Prefix:</b> <code>{{.Config.GraphDBUserPrefix}}</code><br />
|
||||||
<b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
|
<b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
|
||||||
<hr />
|
<hr />
|
||||||
<b>Bookkeeping Database:</b> <code>{{.Config.DistilleryBookkeepingDatabase}}</code><br />
|
<b>Bookkeeping Database:</b> <code>{{.Config.DistilleryDatabase}}</code><br />
|
||||||
<b>Bookkeeping Table:</b> <code>{{.Config.DistilleryBookkeepingTable}}</code><br />
|
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<h2 id="instances">Instances</h2>
|
<h2 id="instances">Instances</h2>
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,9 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
|
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
|
||||||
"github.com/tkw1536/goprogram/stream"
|
"github.com/tkw1536/goprogram/stream"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
@ -98,7 +98,7 @@ func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
|
||||||
type disInstance struct {
|
type disInstance struct {
|
||||||
Time time.Time
|
Time time.Time
|
||||||
|
|
||||||
Instance bookkeeping.Instance
|
Instance models.Instance
|
||||||
Info instances.Info
|
Info instances.Info
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,10 @@ package instances
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
|
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
|
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
"github.com/tkw1536/goprogram/exit"
|
"github.com/tkw1536/goprogram/exit"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
|
|
@ -36,17 +36,17 @@ var errSQL = exit.Error{
|
||||||
// It the WissKI does not exist, returns ErrWissKINotFound.
|
// It the WissKI does not exist, returns ErrWissKINotFound.
|
||||||
func (instances *Instances) WissKI(slug string) (i WissKI, err error) {
|
func (instances *Instances) WissKI(slug string) (i WissKI, err error) {
|
||||||
sql := instances.SQL
|
sql := instances.SQL
|
||||||
if err := sql.Wait(); err != nil {
|
if err := sql.WaitQueryTable(); err != nil {
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
table, err := sql.OpenBookkeeping(false)
|
table, err := sql.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return i, err
|
return i, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// find the instance by slug
|
// find the instance by slug
|
||||||
query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
|
query := table.Where(&models.Instance{Slug: slug}).Find(&i.Instance)
|
||||||
switch {
|
switch {
|
||||||
case query.Error != nil:
|
case query.Error != nil:
|
||||||
return i, errSQL.WithMessageF(query.Error)
|
return i, errSQL.WithMessageF(query.Error)
|
||||||
|
|
@ -62,11 +62,11 @@ func (instances *Instances) WissKI(slug string) (i WissKI, err error) {
|
||||||
// It does not perform any checks on the WissKI itself.
|
// It does not perform any checks on the WissKI itself.
|
||||||
func (instances *Instances) Has(slug string) (ok bool, err error) {
|
func (instances *Instances) Has(slug string) (ok bool, err error) {
|
||||||
sql := instances.SQL
|
sql := instances.SQL
|
||||||
if err := sql.Wait(); err != nil {
|
if err := sql.WaitQueryTable(); err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
table, err := sql.OpenBookkeeping(false)
|
table, err := sql.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
@ -106,12 +106,12 @@ func (instances *Instances) Load(slugs ...string) ([]WissKI, error) {
|
||||||
// find finds instances based on the provided query
|
// find finds instances based on the provided query
|
||||||
func (instances *Instances) find(order bool, query func(table *gorm.DB) *gorm.DB) (results []WissKI, err error) {
|
func (instances *Instances) find(order bool, query func(table *gorm.DB) *gorm.DB) (results []WissKI, err error) {
|
||||||
sql := instances.SQL
|
sql := instances.SQL
|
||||||
if err := sql.Wait(); err != nil {
|
if err := sql.WaitQueryTable(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// open the bookkeeping table
|
// open the bookkeeping table
|
||||||
table, err := sql.OpenBookkeeping(false)
|
table, err := sql.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -126,7 +126,7 @@ func (instances *Instances) find(order bool, query func(table *gorm.DB) *gorm.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetch bookkeeping instances
|
// fetch bookkeeping instances
|
||||||
var bks []bookkeeping.Instance
|
var bks []models.Instance
|
||||||
find = find.Find(&bks)
|
find = find.Find(&bks)
|
||||||
if find.Error != nil {
|
if find.Error != nil {
|
||||||
return nil, errSQL.WithMessageF(find.Error)
|
return nil, errSQL.WithMessageF(find.Error)
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,11 @@
|
||||||
package instances
|
package instances
|
||||||
|
|
||||||
import (
|
import "github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
|
|
||||||
)
|
|
||||||
|
|
||||||
// WissKI represents a single WissKI Instance
|
// WissKI represents a single WissKI Instance
|
||||||
type WissKI struct {
|
type WissKI struct {
|
||||||
// Whatever is stored inside the bookkeeping database
|
// Whatever is stored inside the bookkeeping database
|
||||||
bookkeeping.Instance
|
models.Instance
|
||||||
|
|
||||||
// Credentials to Drupal
|
// Credentials to Drupal
|
||||||
DrupalUsername string
|
DrupalUsername string
|
||||||
|
|
@ -19,7 +17,7 @@ type WissKI struct {
|
||||||
|
|
||||||
// Save saves this instance in the bookkeeping table
|
// Save saves this instance in the bookkeeping table
|
||||||
func (wisski *WissKI) Save() error {
|
func (wisski *WissKI) Save() error {
|
||||||
db, err := wisski.instances.SQL.OpenBookkeeping(false)
|
db, err := wisski.instances.SQL.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -35,7 +33,7 @@ func (wisski *WissKI) Save() error {
|
||||||
|
|
||||||
// Delete deletes this instance from the bookkeeping table
|
// Delete deletes this instance from the bookkeeping table
|
||||||
func (wisski *WissKI) Delete() error {
|
func (wisski *WissKI) Delete() error {
|
||||||
db, err := wisski.instances.SQL.OpenBookkeeping(false)
|
db, err := wisski.instances.SQL.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
139
internal/component/sql/connect.go
Normal file
139
internal/component/sql/connect.go
Normal file
|
|
@ -0,0 +1,139 @@
|
||||||
|
package sql
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
mysqldriver "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
|
"gorm.io/driver/mysql"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
|
||||||
|
)
|
||||||
|
|
||||||
|
//
|
||||||
|
// ========== low-level connection ==========
|
||||||
|
//
|
||||||
|
|
||||||
|
// Query performs a database query, outside a database contect
|
||||||
|
func (sql *SQL) Query(query string, args ...interface{}) error {
|
||||||
|
// connect to the server
|
||||||
|
conn, err := sql.connect("")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// do the query!
|
||||||
|
{
|
||||||
|
_, err := conn.Exec(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitQuery waits for the query interface to be able to connect to the database
|
||||||
|
func (sql *SQL) WaitQuery() error {
|
||||||
|
return wait.Wait(func() bool {
|
||||||
|
err := sql.Query("select 1;")
|
||||||
|
// log.Printf("[WaitQuery] %s\n", err) // debug
|
||||||
|
return err == nil
|
||||||
|
}, sql.PollInterval, sql.PollContext)
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// ========== connection via gorm ==========
|
||||||
|
//
|
||||||
|
|
||||||
|
// QueryTable returns a gorm.DB to connect to the provided gorm database table
|
||||||
|
func (sql *SQL) QueryTable(silent bool, name string) (*gorm.DB, error) {
|
||||||
|
conn, err := sql.connect(sql.Config.DistilleryDatabase)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// gorm configuration
|
||||||
|
config := &gorm.Config{}
|
||||||
|
if silent {
|
||||||
|
config.Logger = logger.Default.LogMode(logger.Silent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mysql connection
|
||||||
|
cfg := mysql.Config{
|
||||||
|
Conn: conn,
|
||||||
|
|
||||||
|
DefaultStringSize: 256,
|
||||||
|
}
|
||||||
|
|
||||||
|
// open the gorm connection!
|
||||||
|
db, err := gorm.Open(mysql.New(cfg), config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// set the table
|
||||||
|
db = db.Table(name)
|
||||||
|
|
||||||
|
// check that nothing went wrong
|
||||||
|
if db.Error != nil {
|
||||||
|
return nil, db.Error
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitQueryTable waits for a connection to succeed via QueryTable
|
||||||
|
func (sql *SQL) WaitQueryTable() error {
|
||||||
|
return wait.Wait(func() bool {
|
||||||
|
_, err := sql.QueryTable(true, models.InstanceTable)
|
||||||
|
return err == nil
|
||||||
|
}, sql.PollInterval, sql.PollContext)
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// ========== low-level database connection ==========
|
||||||
|
//
|
||||||
|
|
||||||
|
func (ssql *SQL) connect(database string) (*sql.DB, error) {
|
||||||
|
conn, err := sql.Open("mysql", ssql.dsn(database))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
conn.SetMaxIdleConns(0)
|
||||||
|
|
||||||
|
return conn, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dsn returns a dsn fof connecting to the database
|
||||||
|
func (sql *SQL) dsn(database string) string {
|
||||||
|
user := sql.Config.MysqlAdminUser
|
||||||
|
pass := sql.Config.MysqlAdminPassword
|
||||||
|
network := sql.network()
|
||||||
|
server := sql.ServerURL
|
||||||
|
|
||||||
|
return fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8&parseTime=True&loc=Local", user, pass, network, server, database)
|
||||||
|
}
|
||||||
|
|
||||||
|
var proxyNameCounter uint64
|
||||||
|
|
||||||
|
// network returns the network to use to connect to the database
|
||||||
|
func (sql *SQL) network() string {
|
||||||
|
return sql.lazyNetwork.Get(func() (name string) {
|
||||||
|
network := "tcp"
|
||||||
|
|
||||||
|
// register a new DialContext function to use the environment.
|
||||||
|
// this seems like a bit of a hack, but it works for now.
|
||||||
|
name = fmt.Sprintf("sql-network-%d", atomic.AddUint64(&proxyNameCounter, 1))
|
||||||
|
mysqldriver.RegisterDialContext(name, func(ctx context.Context, addr string) (net.Conn, error) {
|
||||||
|
return sql.Core.Environment.DialContext(ctx, network, addr)
|
||||||
|
})
|
||||||
|
return
|
||||||
|
})
|
||||||
|
}
|
||||||
1
internal/component/sql/connect_shell.go
Normal file
1
internal/component/sql/connect_shell.go
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
package sql
|
||||||
|
|
@ -1,111 +0,0 @@
|
||||||
package sql
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"sync/atomic"
|
|
||||||
|
|
||||||
mysqldriver "github.com/go-sql-driver/mysql"
|
|
||||||
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
|
|
||||||
"github.com/tkw1536/goprogram/stream"
|
|
||||||
"gorm.io/driver/mysql"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
var proxyNameCounter uint64
|
|
||||||
|
|
||||||
// network returns the network to use to connect to the database
|
|
||||||
func (sql *SQL) network() string {
|
|
||||||
return sql.lazyNetwork.Get(func() (name string) {
|
|
||||||
network := "tcp"
|
|
||||||
|
|
||||||
// register a new DialContext function to use the environment.
|
|
||||||
// this seems like a bit of a hack, but it works for now.
|
|
||||||
name = fmt.Sprintf("sql-network-%d", atomic.AddUint64(&proxyNameCounter, 1))
|
|
||||||
mysqldriver.RegisterDialContext(name, func(ctx context.Context, addr string) (net.Conn, error) {
|
|
||||||
return sql.Core.Environment.DialContext(ctx, network, addr)
|
|
||||||
})
|
|
||||||
return
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// sqlOpen opens a new sql connection to the provided database using the administrative credentials
|
|
||||||
func (sql *SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
|
|
||||||
cfg := mysql.Config{
|
|
||||||
DriverName: "mysql",
|
|
||||||
DSN: fmt.Sprintf("%s:%s@%s(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.network(), sql.ServerURL, database),
|
|
||||||
DefaultStringSize: 256,
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := gorm.Open(mysql.New(cfg), config)
|
|
||||||
if err != nil {
|
|
||||||
return db, err
|
|
||||||
}
|
|
||||||
|
|
||||||
gdb, err := db.DB()
|
|
||||||
if err != nil {
|
|
||||||
return db, err
|
|
||||||
}
|
|
||||||
gdb.SetMaxIdleConns(0)
|
|
||||||
|
|
||||||
return db, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// OpenBookkeeping opens a connection to the bookkeeping database
|
|
||||||
func (sql *SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
|
|
||||||
|
|
||||||
config := &gorm.Config{}
|
|
||||||
if silent {
|
|
||||||
config.Logger = logger.Default.LogMode(logger.Silent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// open the database
|
|
||||||
db, err := sql.openDatabase(sql.Config.DistilleryBookkeepingDatabase, config)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// load the table
|
|
||||||
table := db.Table(sql.Config.DistilleryBookkeepingTable)
|
|
||||||
if table.Error != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return table, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shell runs a mysql shell command.
|
|
||||||
func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) {
|
|
||||||
return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WaitShell waits for the sql database to be reachable via shell
|
|
||||||
func (sql *SQL) WaitShell() error {
|
|
||||||
n := stream.FromNil()
|
|
||||||
return wait.Wait(func() bool {
|
|
||||||
code, err := sql.Shell(n, "-e", "show databases;")
|
|
||||||
return err == nil && code == 0
|
|
||||||
}, sql.PollInterval, sql.PollContext)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait waits for a connection to the bookkeeping table to suceed
|
|
||||||
func (sql *SQL) Wait() error {
|
|
||||||
return wait.Wait(func() bool {
|
|
||||||
_, err := sql.OpenBookkeeping(true)
|
|
||||||
return err == nil
|
|
||||||
}, sql.PollInterval, sql.PollContext)
|
|
||||||
}
|
|
||||||
|
|
||||||
var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
|
|
||||||
|
|
||||||
// Query performs a raw database query
|
|
||||||
func (sql *SQL) Query(query string, args ...interface{}) bool {
|
|
||||||
raw := sqle.Format(query, args...)
|
|
||||||
code, err := sql.Shell(stream.FromNil(), "-e", raw)
|
|
||||||
return err == nil && code == 0
|
|
||||||
}
|
|
||||||
|
|
@ -6,47 +6,81 @@ import (
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SQLProvision provisions a new sql database and user
|
var errProvisionInvalidDatabaseParams = errors.New("Provision: Invalid parameters")
|
||||||
|
var errProvisionInvalidGrant = errors.New("Provision: Grant failed")
|
||||||
|
|
||||||
|
// Provision provisions a new sql database and user
|
||||||
func (sql *SQL) Provision(name, user, password string) error {
|
func (sql *SQL) Provision(name, user, password string) error {
|
||||||
// wait for the database
|
|
||||||
if err := sql.WaitShell(); err != nil {
|
// NOTE(twiesing): We shouldn't use string concat to build sql queries.
|
||||||
|
// But the driver doesn't support using query params for this particular query.
|
||||||
|
// Apparently it's a "feature", see https://github.com/go-sql-driver/mysql/issues/398#issuecomment-169951763.
|
||||||
|
|
||||||
|
// quick and dirty check to make sure that all the names won't sql inject.
|
||||||
|
if !sqle.IsSafeDatabaseLiteral(name) || !sqle.IsSafeDatabaseSingleQuote(user) || !sqle.IsSafeDatabaseSingleQuote(password) {
|
||||||
|
return errProvisionInvalidDatabaseParams
|
||||||
|
}
|
||||||
|
|
||||||
|
// We use the sql shell here, because not only can we not use query params, but the driver outright rejects queries.
|
||||||
|
// Queries of the form "CREATE USER 'test'@'%' IDENTIFIED BY 'test'; FLUSH PRIVILEGES;" return error 1064 when using driver, but are fine with the shell.
|
||||||
|
// This should be fixed eventually, but I have no idea how.
|
||||||
|
|
||||||
|
if err := sql.unsafeWaitShell(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// it's not a safe database name!
|
query := "CREATE DATABASE `" + name + "`;" +
|
||||||
if !sqle.IsSafeDatabaseName(name) {
|
"CREATE USER '" + user + "'@'%' IDENTIFIED BY '" + password + "';" +
|
||||||
return errInvalidDatabaseName
|
"GRANT ALL PRIVILEGES ON `" + name + "`.* TO `" + user + "`@`%`; FLUSH PRIVILEGES;"
|
||||||
|
if !sql.unsafeQueryShell(query) {
|
||||||
|
return errProvisionInvalidGrant
|
||||||
}
|
}
|
||||||
|
|
||||||
// create the database and user!
|
|
||||||
if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
|
|
||||||
return errors.New("SQLProvision: Failed to create user")
|
|
||||||
}
|
|
||||||
|
|
||||||
// and done!
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var errSQLPurgeUser = errors.New("unable to delete user")
|
var errCreateSuperuserGrant = errors.New("CreateSuperUser: Grant failed")
|
||||||
|
|
||||||
|
func (sql *SQL) CreateSuperuser(user, password string, allowExisting bool) error {
|
||||||
|
// NOTE(twiesing): This function unsafely uses the shell directly to create a superuser.
|
||||||
|
// This is for two reasons:
|
||||||
|
// (1) this is used during bootstraping
|
||||||
|
// (2) The underlying driver doesn't support "GRANT ALL PRIVILEGES"
|
||||||
|
// See also [sql.Provision].
|
||||||
|
|
||||||
|
if !sqle.IsSafeDatabaseSingleQuote(user) || !sqle.IsSafeDatabaseSingleQuote(password) {
|
||||||
|
return errProvisionInvalidDatabaseParams
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := sql.unsafeWaitShell(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var IfNotExists string
|
||||||
|
if allowExisting {
|
||||||
|
IfNotExists = "IF NOT EXISTS"
|
||||||
|
}
|
||||||
|
|
||||||
|
query := "CREATE USER " + IfNotExists + " '" + user + "'@'%' IDENTIFIED BY '" + password + "';" +
|
||||||
|
"GRANT ALL PRIVILEGES ON *.* TO '" + user + "'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;"
|
||||||
|
if !sql.unsafeQueryShell(query) {
|
||||||
|
return errCreateSuperuserGrant
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// SQLPurgeUser deletes the specified user from the database
|
// SQLPurgeUser deletes the specified user from the database
|
||||||
func (sql *SQL) PurgeUser(user string) error {
|
func (sql *SQL) PurgeUser(user string) error {
|
||||||
if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
|
return sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user)
|
||||||
return errSQLPurgeUser
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var errSQLPurgeDB = errors.New("unable to drop database")
|
var errSQLPurgeDB = errors.New("unable to drop database: unsafe database name")
|
||||||
|
|
||||||
// SQLPurgeDatabase deletes the specified db from the database
|
// SQLPurgeDatabase deletes the specified db from the database
|
||||||
func (sql *SQL) PurgeDatabase(db string) error {
|
func (sql *SQL) PurgeDatabase(db string) error {
|
||||||
if !sqle.IsSafeDatabaseName(db) {
|
if !sqle.IsSafeDatabaseLiteral(db) {
|
||||||
return errSQLPurgeDB
|
return errSQLPurgeDB
|
||||||
}
|
}
|
||||||
if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") {
|
return sql.Query("DROP DATABASE IF EXISTS `" + db + "`")
|
||||||
return errSQLPurgeDB
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,57 +4,82 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
|
||||||
"github.com/tkw1536/goprogram/stream"
|
"github.com/tkw1536/goprogram/stream"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Shell runs a mysql shell with the provided databases.
|
||||||
|
//
|
||||||
|
// NOTE(twiesing): This command should not be used to connect to the database or execute queries except in known situations.
|
||||||
|
func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) {
|
||||||
|
return sql.Stack(sql.Environment).Exec(io, "sql", "mysql", argv...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unsafeWaitShell waits for a connection via the database shell to succeed
|
||||||
|
func (sql *SQL) unsafeWaitShell() error {
|
||||||
|
n := stream.FromNil()
|
||||||
|
return wait.Wait(func() bool {
|
||||||
|
code, err := sql.Shell(n, "-e", "select 1;")
|
||||||
|
// log.Printf("[unsafeWaitShell] %d %s\n", code, err) // debug
|
||||||
|
return err == nil && code == 0
|
||||||
|
}, sql.PollInterval, sql.PollContext)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unsafeQuery shell executes a raw database query.
|
||||||
|
func (sql *SQL) unsafeQueryShell(query string) bool {
|
||||||
|
code, err := sql.Shell(stream.FromNil(), "-e", query)
|
||||||
|
return err == nil && code == 0
|
||||||
|
}
|
||||||
|
|
||||||
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
|
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
|
||||||
var errSQLUnsafeDatabaseName = errors.New("bookkeeping database has an unsafe name")
|
var errSQLUnsafeDatabaseName = errors.New("distillery database has an unsafe name")
|
||||||
var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
|
|
||||||
|
|
||||||
// Update initializes or updates the SQL database.
|
// Update initializes or updates the SQL database.
|
||||||
func (sql *SQL) Update(io stream.IOStream) error {
|
func (sql *SQL) Update(io stream.IOStream) error {
|
||||||
if err := sql.WaitShell(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// create the admin user
|
// unsafely create the admin user!
|
||||||
logging.LogMessage(io, "Creating administrative user")
|
|
||||||
{
|
{
|
||||||
username := sql.Config.MysqlAdminUser
|
if err := sql.unsafeWaitShell(); err != nil {
|
||||||
password := sql.Config.MysqlAdminPassword
|
return err
|
||||||
if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
|
}
|
||||||
return errSQLUnableToCreateUser
|
logging.LogMessage(io, "Creating administrative user")
|
||||||
|
{
|
||||||
|
username := sql.Config.MysqlAdminUser
|
||||||
|
password := sql.Config.MysqlAdminPassword
|
||||||
|
if err := sql.CreateSuperuser(username, password, true); err != nil {
|
||||||
|
return errSQLUnableToCreateUser
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// create the admin user
|
// create the admin user
|
||||||
logging.LogMessage(io, "Creating sql database")
|
logging.LogMessage(io, "Creating sql database")
|
||||||
{
|
{
|
||||||
if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) {
|
if !sqle.IsSafeDatabaseLiteral(sql.Config.DistilleryDatabase) {
|
||||||
return errSQLUnsafeDatabaseName
|
return errSQLUnsafeDatabaseName
|
||||||
}
|
}
|
||||||
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase)
|
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryDatabase)
|
||||||
if !sql.Query(createDBSQL) {
|
if err := sql.Query(createDBSQL); err != nil {
|
||||||
return errSQLUnableToCreate
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// wait for the database to come up
|
// wait for the database to come up
|
||||||
logging.LogMessage(io, "Waiting for database update to be complete")
|
logging.LogMessage(io, "Waiting for database update to be complete")
|
||||||
sql.Wait()
|
sql.WaitQueryTable()
|
||||||
|
|
||||||
// open the database
|
// open the database
|
||||||
logging.LogMessage(io, "Migrating bookkeeping table")
|
logging.LogMessage(io, "Migrating instances table")
|
||||||
{
|
{
|
||||||
db, err := sql.OpenBookkeeping(false)
|
db, err := sql.QueryTable(false, models.InstanceTable)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to access bookkeeping table: %s", err)
|
return fmt.Errorf("unable to access bookkeeping table: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
|
if err := db.AutoMigrate(&models.Instance{}); err != nil {
|
||||||
return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
|
return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -57,8 +57,7 @@ type Config struct {
|
||||||
|
|
||||||
// In addition to the filesystem the WissKI distillery requires a single SQL table.
|
// In addition to the filesystem the WissKI distillery requires a single SQL table.
|
||||||
// It uses this database to store a list of installed things.
|
// It uses this database to store a list of installed things.
|
||||||
DistilleryBookkeepingDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" parser:"slug"`
|
DistilleryDatabase string `env:"DISTILLERY_BOOKKEEPING_DATABASE" default:"distillery" parser:"slug"`
|
||||||
DistilleryBookkeepingTable string `env:"DISTILLERY_BOOKKEEPING_TABLE" default:"distillery" parser:"slug"`
|
|
||||||
|
|
||||||
// Various components use password-based-authentication.
|
// Various components use password-based-authentication.
|
||||||
// These passwords are generated automatically.
|
// These passwords are generated automatically.
|
||||||
|
|
|
||||||
|
|
@ -40,11 +40,9 @@ MYSQL_DATABASE_PREFIX=mysql-factory-
|
||||||
GRAPHDB_USER_PREFIX=graphdb-factory-
|
GRAPHDB_USER_PREFIX=graphdb-factory-
|
||||||
GRAPHDB_REPO_PREFIX=graphdb-factory-
|
GRAPHDB_REPO_PREFIX=graphdb-factory-
|
||||||
|
|
||||||
# In addition to the filesystem the WissKI distillery requires a single SQL table.
|
# In addition to the filesystem the WissKI distillery requires a 'bookkeeping' database.
|
||||||
# It uses this database to store a list of installed things
|
# This is used to store several settings.
|
||||||
DISTILLERY_BOOKKEEPING_DATABASE=distillery
|
DISTILLERY_BOOKKEEPING_DATABASE=distillery
|
||||||
DISTILLERY_BOOKKEEPING_TABLE=distillery
|
|
||||||
|
|
||||||
|
|
||||||
# Various components use password-based-authentication.
|
# Various components use password-based-authentication.
|
||||||
# These passwords are generated automatically.
|
# These passwords are generated automatically.
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
// Package bookkeeping implements reading and writing from the bookkeeping table
|
package models
|
||||||
package bookkeeping
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql/driver"
|
"database/sql/driver"
|
||||||
|
|
@ -7,11 +6,15 @@ import (
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Instance is a WissKI Instance inside the bookkeeping table.
|
// InstanceTable is the name of the table the 'Instance' model is stored in.
|
||||||
|
const InstanceTable = "distillery"
|
||||||
|
|
||||||
|
// Instance is a WissKI Instance stored inside the sql database.
|
||||||
|
//
|
||||||
// It does not represent a running instance; it does not perform any validation.
|
// It does not represent a running instance; it does not perform any validation.
|
||||||
type Instance struct {
|
type Instance struct {
|
||||||
// NOTE: Modifying this struct requires a database migration.
|
// NOTE: Modifying this struct requires a database migration.
|
||||||
// This should nnever be done unless you know what you're doing.
|
// This should never be done unless you know what you're doing.
|
||||||
|
|
||||||
// Primary key for the instance
|
// Primary key for the instance
|
||||||
Pk uint `gorm:"column:pk;primaryKey"`
|
Pk uint `gorm:"column:pk;primaryKey"`
|
||||||
|
|
@ -8,8 +8,8 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
|
|
||||||
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
|
||||||
|
"github.com/FAU-CDI/wisski-distillery/internal/models"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
"github.com/FAU-CDI/wisski-distillery/pkg/countwriter"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
|
||||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||||
|
|
@ -82,7 +82,7 @@ type SnapshotDescription struct {
|
||||||
// Snapshot represents the result of generating a snapshot
|
// Snapshot represents the result of generating a snapshot
|
||||||
type Snapshot struct {
|
type Snapshot struct {
|
||||||
Description SnapshotDescription
|
Description SnapshotDescription
|
||||||
Instance bookkeeping.Instance
|
Instance models.Instance
|
||||||
|
|
||||||
// Start and End Time of the snapshot
|
// Start and End Time of the snapshot
|
||||||
StartTime time.Time
|
StartTime time.Time
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ const ExecCommandError = 127
|
||||||
const DefaultFilePerm fs.FileMode = 0666
|
const DefaultFilePerm fs.FileMode = 0666
|
||||||
|
|
||||||
// DefaultDirPerm is the default mode to use for directories
|
// DefaultDirPerm is the default mode to use for directories
|
||||||
const DefaultDirPerm fs.FileMode = os.ModeDir & fs.ModePerm
|
const DefaultDirPerm fs.FileMode = fs.ModeDir | fs.ModePerm
|
||||||
|
|
||||||
// IsExist checks if the provided error represents a 'does not exist' errror
|
// IsExist checks if the provided error represents a 'does not exist' errror
|
||||||
func IsExist(err error) bool {
|
func IsExist(err error) bool {
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,13 @@ import (
|
||||||
"unicode"
|
"unicode"
|
||||||
)
|
)
|
||||||
|
|
||||||
// IsSafeDatabaseName checks if a string is safe to be used as a database name
|
// IsSafeDatabaseSingleQuote checks if value can safely be put inside 's inside a database query
|
||||||
func IsSafeDatabaseName(value string) bool {
|
func IsSafeDatabaseSingleQuote(value string) bool {
|
||||||
|
return !strings.ContainsAny(value, "'`") // TODO: This should be safer, but it's relatively controlled
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsSafeDatabaseLiteral checks if a value is safe to be used as a database query literal
|
||||||
|
func IsSafeDatabaseLiteral(value string) bool {
|
||||||
// the empty name is not allowed!
|
// the empty name is not allowed!
|
||||||
if len(value) == 0 {
|
if len(value) == 0 {
|
||||||
return false
|
return false
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ import (
|
||||||
"github.com/feiin/sqlstring"
|
"github.com/feiin/sqlstring"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TODO: This is really unsafe and shouldn't be used at all.
|
||||||
|
|
||||||
// Format formats the provided query with the given parameters.
|
// Format formats the provided query with the given parameters.
|
||||||
func Format(query string, params ...interface{}) string {
|
func Format(query string, params ...interface{}) string {
|
||||||
return sqlstring.Format(query, params...)
|
return sqlstring.Format(query, params...)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue