rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sql: Refactor use of tables

Browse source
This commit is contained in:
Tom Wiesing 2023-01-09 10:24:24 +01:00
parent 85fe5b5c5a
commit 73d821e320
No known key found for this signature in database
19 changed files with 191 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

1
internal/dis/component/auth/auth.go
View file

@ -25,6 +25,7 @@ type Auth struct {
var (
_ component.Routeable = (*Auth)(nil)
_ component.Table = (*Auth)(nil)
)
func (auth *Auth) Routes() component.Routes {

13
internal/dis/component/auth/policy/policy.go
View file

@ -7,6 +7,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/auth"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/tkw1536/goprogram/lib/reflectx"
"gorm.io/gorm"
)
@ -22,8 +23,16 @@ type Policy struct {
var (
_ component.Provisionable = (*Policy)(nil)
_ component.UserDeleteHook = (*Policy)(nil)
_ component.Table = (*Policy)(nil)
)
func (pol *Policy) table(ctx context.Context) (*gorm.DB, error) {
return pol.Dependencies.SQL.QueryTable(ctx, true, models.GrantTable)
func (pol *Policy) TableInfo() component.TableInfo {
return component.TableInfo{
Name: models.GrantTable,
Model: reflectx.TypeOf[models.Grant](),
}
}
func (pol *Policy) table(ctx context.Context) (*gorm.DB, error) {
return pol.Dependencies.SQL.QueryTable(ctx, pol)
}

22
internal/dis/component/auth/user.go
View file

@ -7,20 +7,29 @@ import (
"fmt"
"image/png"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/pkg/errors"
"github.com/pquerna/otp"
"github.com/pquerna/otp/totp"
"github.com/tkw1536/goprogram/lib/reflectx"
"golang.org/x/crypto/bcrypt"
)
// ErrUserNotFound is returned when a user is not found
var ErrUserNotFound = errors.New("user not found")
func (auth *Auth) TableInfo() component.TableInfo {
return component.TableInfo{
Name: models.UserTable,
Model: reflectx.TypeOf[models.User](),
}
}
// Users returns all users in the database
func (auth *Auth) Users(ctx context.Context) (users []*AuthUser, err error) {
// query the user table
table, err := auth.Dependencies.SQL.QueryTable(ctx, false, models.UserTable)
table, err := auth.Dependencies.SQL.QueryTable(ctx, auth)
if err != nil {
return
}
@ -53,7 +62,10 @@ func (auth *Auth) User(ctx context.Context, name string) (user *AuthUser, err er
}
// return the user
table, err := auth.Dependencies.SQL.QueryTable(ctx, false, models.UserTable)
table, err := auth.Dependencies.SQL.QueryTable(ctx, auth)
if err != nil {
return
}
if err != nil {
return
}
@ -81,7 +93,7 @@ func (auth *Auth) User(ctx context.Context, name string) (user *AuthUser, err er
// The user is not associated to any WissKIs, and has no password set.
func (auth *Auth) CreateUser(ctx context.Context, name string) (user *AuthUser, err error) {
// return the user
table, err := auth.Dependencies.SQL.QueryTable(ctx, false, models.UserTable)
table, err := auth.Dependencies.SQL.QueryTable(ctx, auth)
if err != nil {
return
}
@ -267,7 +279,7 @@ func (au *AuthUser) MakeRegular(ctx context.Context) error {
// Save saves the given user in the database
func (au *AuthUser) Save(ctx context.Context) error {
table, err := au.auth.Dependencies.SQL.QueryTable(ctx, false, models.UserTable)
table, err := au.auth.Dependencies.SQL.QueryTable(ctx, au.auth)
if err != nil {
return err
}
@ -276,7 +288,7 @@ func (au *AuthUser) Save(ctx context.Context) error {
// Delete deletes the user from the database
func (au *AuthUser) Delete(ctx context.Context) error {
table, err := au.auth.Dependencies.SQL.QueryTable(ctx, false, models.UserTable)
table, err := au.auth.Dependencies.SQL.QueryTable(ctx, au.auth)
if err != nil {
return err
}

16
internal/dis/component/exporter/logger/logger.go
View file

@ -9,6 +9,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/status"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/lib/reflectx"
)
// Logger is responsible for logging backups and snapshots
@ -19,6 +20,17 @@ type Logger struct {
}
}
var (
_ component.Table = (*Logger)(nil)
)
func (*Logger) TableInfo() component.TableInfo {
return component.TableInfo{
Model: reflectx.TypeOf[models.Export](),
Name: models.ExportTable,
}
}
// For retrieves (and prunes) the ExportLog.
// Slug determines if entries for Backups (empty slug)
// or a specific Instance (non-empty slug) are returned.
@ -36,7 +48,7 @@ func (log *Logger) For(ctx context.Context, slug string) (exports []models.Expor
// Log retrieves (and prunes) all entries in the snapshot log.
func (log *Logger) Log(ctx context.Context) ([]models.Export, error) {
// query the table!
table, err := log.Dependencies.SQL.QueryTable(ctx, false, models.ExportTable)
table, err := log.Dependencies.SQL.QueryTable(ctx, log)
if err != nil {
return nil, err
}
@ -68,7 +80,7 @@ func (log *Logger) Log(ctx context.Context) ([]models.Export, error) {
// AddToExportLog adds the provided export to the log.
func (log *Logger) Add(ctx context.Context, export models.Export) error {
// find the table
table, err := log.Dependencies.SQL.QueryTable(ctx, false, models.ExportTable)
table, err := log.Dependencies.SQL.QueryTable(ctx, log)
if err != nil {
return err
}

8
internal/dis/component/instances/instances.go
View file

@ -22,6 +22,8 @@ type Instances struct {
Dependencies struct {
Malt *malt.Malt
SQL *sql.SQL
InstanceTable *sql.InstanceTable
}
}
@ -50,7 +52,7 @@ func (instances *Instances) WissKI(ctx context.Context, slug string) (wissKI *wi
return nil, err
}
table, err := sql.QueryTable(ctx, false, models.InstanceTable)
table, err := sql.QueryTable(ctx, instances.Dependencies.InstanceTable)
if err != nil {
return nil, err
}
@ -90,7 +92,7 @@ func (instances *Instances) Has(ctx context.Context, slug string) (ok bool, err
return false, err
}
table, err := sql.QueryTable(ctx, false, models.InstanceTable)
table, err := sql.QueryTable(ctx, instances.Dependencies.InstanceTable)
if err != nil {
return false, err
}
@ -135,7 +137,7 @@ func (instances *Instances) find(ctx context.Context, order bool, query func(tab
}
// open the bookkeeping table
table, err := sql.QueryTable(ctx, false, models.InstanceTable)
table, err := sql.QueryTable(ctx, instances.Dependencies.InstanceTable)
if err != nil {
return nil, err
}

5
internal/dis/component/instances/malt/malt.go
View file

@ -13,8 +13,11 @@ import (
type Malt struct {
component.Base
SQL *sql.SQL `auto:"true"`
InstanceTable *sql.InstanceTable `auto:"true"`
LockTable *sql.LockTable `auto:"true"`
TS *triplestore.Triplestore `auto:"true"`
SQL *sql.SQL `auto:"true"`
Meta *meta.Meta `auto:"true"`
ExporterLog *logger.Logger `auto:"true"`
Policy *policy.Policy `auto:"true"`

15
internal/dis/component/meta/meta.go
View file

@ -5,6 +5,8 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/tkw1536/goprogram/lib/reflectx"
)
// Component meta is responsible for managing metadata per WissKI Instance
@ -20,8 +22,16 @@ type Meta struct {
var (
_ component.Provisionable = (*Meta)(nil)
_ component.Table = (*Meta)(nil)
)
func (*Meta) TableInfo() component.TableInfo {
return component.TableInfo{
Model: reflectx.TypeOf[models.Metadatum](),
Name: models.MetadataTable,
}
}
// Storage returns a Storage for the instance with the given slug.
// When slug is nil, returns a global storage.
func (meta *Meta) Storage(slug string) *Storage {
@ -40,8 +50,9 @@ func (meta *Meta) Storage(slug string) *Storage {
// create a new storage
meta.sc[slug] = &Storage{
Slug: slug,
sql: meta.Dependencies.SQL,
Slug: slug,
sql: meta.Dependencies.SQL,
table: meta,
}
return meta.sc[slug]
}

17
internal/dis/component/meta/storage.go
View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/tkw1536/goprogram/lib/collection"
@ -20,13 +21,15 @@ var ErrMetadatumNotSet = errors.New("metadatum not set")
// Storage manages metadata for either the entire distillery, or a single slug
type Storage struct {
Slug string
sql *sql.SQL
table component.Table
sql *sql.SQL
}
// Get retrieves metadata with the provided key and deserializes the first one into target.
// If no metadatum exists, returns [ErrMetadatumNotSet].
func (s Storage) Get(ctx context.Context, key Key, target any) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}
@ -55,7 +58,7 @@ func (s Storage) Get(ctx context.Context, key Key, target any) error {
//
// When no metadatum exists, targets is not called, and nil error is returned.
func (s Storage) GetAll(ctx context.Context, key Key, target func(index, total int) any) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}
@ -82,7 +85,7 @@ func (s Storage) GetAll(ctx context.Context, key Key, target func(index, total i
// Delete deletes all metadata with the provided key.
func (s Storage) Delete(ctx context.Context, key Key) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}
@ -98,7 +101,7 @@ func (s Storage) Delete(ctx context.Context, key Key) error {
// Set serializes value and stores it with the provided key.
// Any other metadata with the same key is deleted.
func (s Storage) Set(ctx context.Context, key Key, value any) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}
@ -133,7 +136,7 @@ func (s Storage) Set(ctx context.Context, key Key, value any) error {
// Set serializes values and stores them with the provided key.
// Any other metadata with the same key is deleted.
func (s Storage) SetAll(ctx context.Context, key Key, values ...any) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}
@ -167,7 +170,7 @@ func (s Storage) SetAll(ctx context.Context, key Key, values ...any) error {
// Purge removes all metadata, regardless of key.
func (s Storage) Purge(ctx context.Context) error {
table, err := s.sql.QueryTable(ctx, true, models.MetadataTable)
table, err := s.sql.QueryTable(ctx, s.table)
if err != nil {
return err
}

13
internal/dis/component/sql/connect.go
View file

@ -14,6 +14,7 @@ import (
"gorm.io/gorm"
"gorm.io/gorm/logger"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/timex"
)
@ -44,8 +45,13 @@ func (sql *SQL) Exec(query string, args ...interface{}) error {
// ========== connection via gorm ==========
//
// QueryTable returns a gorm.DB to connect to the provided distillery database table
func (sql *SQL) QueryTable(ctx context.Context, silent bool, table string) (*gorm.DB, error) {
// QueryTable returns a gorm.DB to connect to the provided table of the given model
func (sql *SQL) QueryTable(ctx context.Context, table component.Table) (*gorm.DB, error) {
return sql.queryTable(ctx, false, table.TableInfo().Name)
}
// queryTable returns a gorm.DB to connect to the provided distillery database table
func (sql *SQL) queryTable(ctx context.Context, silent bool, table string) (*gorm.DB, error) {
conn, err := sql.connect(sql.Config.DistilleryDatabase)
if err != nil {
return nil, err
@ -88,7 +94,8 @@ func (sql *SQL) QueryTable(ctx context.Context, silent bool, table string) (*gor
func (sql *SQL) WaitQueryTable(ctx context.Context) error {
// TODO: Establish a convention on when to wait for this!
return timex.TickUntilFunc(func(time.Time) bool {
_, err := sql.QueryTable(ctx, true, models.InstanceTable)
// TODO: Use a different table here
_, err := sql.queryTable(ctx, true, models.InstanceTable)
return err == nil
}, ctx, sql.PollInterval)
}

29
internal/dis/component/sql/logger.go
View file

@ -25,6 +25,23 @@ func newGormLogger() logger.Interface {
}
}
// logLevelMap is the default
var logLevelMap = map[logger.LogLevel]zerolog.Level{
logger.Silent: zerolog.Disabled,
logger.Error: zerolog.ErrorLevel,
logger.Warn: zerolog.WarnLevel,
logger.Info: zerolog.DebugLevel,
logger.Info + 1: zerolog.TraceLevel,
}
func (*gormLogger) NewEvent(ctx context.Context, level logger.LogLevel) *zerolog.Event {
zl, ok := logLevelMap[level]
if !ok {
zl = zerolog.NoLevel
}
return zerolog.Ctx(ctx).WithLevel(zl)
}
func (gl *gormLogger) LogMode(level logger.LogLevel) logger.Interface {
new := *gl
new.Level = level
@ -35,19 +52,19 @@ func (gl *gormLogger) Info(ctx context.Context, format string, v ...interface{})
if gl.Level < logger.Info {
return
}
zerolog.Ctx(ctx).Info().Msgf(format, v...)
gl.NewEvent(ctx, logger.Info).Msgf(format, v...)
}
func (gl *gormLogger) Warn(ctx context.Context, format string, v ...interface{}) {
if gl.Level < logger.Warn {
return
}
zerolog.Ctx(ctx).Warn().Msgf(format, v...)
gl.NewEvent(ctx, logger.Warn).Msgf(format, v...)
}
func (gl *gormLogger) Error(ctx context.Context, format string, v ...interface{}) {
if gl.Level < logger.Error {
return
}
zerolog.Ctx(ctx).Error().Msgf(format, v...)
gl.NewEvent(ctx, logger.Error).Msgf(format, v...)
}
func (gl *gormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
if gl.Level < logger.Silent {
@ -60,15 +77,15 @@ func (gl *gormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql
sql, rows := fc()
src := utils.FileWithLineNum()
zerolog.Ctx(ctx).Err(err).Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msg("GORM")
gl.NewEvent(ctx, logger.Error).Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msg("GORM")
case elapsed > gl.SlowThreshold && gl.SlowThreshold != 0 && gl.Level >= logger.Warn:
sql, rows := fc()
src := utils.FileWithLineNum()
zerolog.Ctx(ctx).Warn().Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msgf("GORM: Slow SQL >= ", gl.SlowThreshold)
gl.NewEvent(ctx, logger.Warn).Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msgf("GORM: Slow SQL >= ", gl.SlowThreshold)
case gl.Level == logger.Info:
sql, rows := fc()
src := utils.FileWithLineNum()
zerolog.Ctx(ctx).Debug().Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msg("GORM")
gl.NewEvent(ctx, logger.Info+1).Str("src", src).Int64("rows", rows).Dur("elapsed", elapsed).Str("sql", sql).Msg("GORM")
}
}

3
internal/dis/component/sql/sql.go
View file

@ -12,6 +12,9 @@ import (
type SQL struct {
component.Base
Dependencies struct {
Tables []component.Table
}
ServerURL string // upstream server url

39
internal/dis/component/sql/tables.go Normal file
View file

@ -0,0 +1,39 @@
package sql
import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/tkw1536/goprogram/lib/reflectx"
)
// This file defines additional tables used by multiple components
type InstanceTable struct {
component.Base
}
var (
_ component.Table = (*InstanceTable)(nil)
)
func (*InstanceTable) TableInfo() component.TableInfo {
return component.TableInfo{
Model: reflectx.TypeOf[models.Instance](),
Name: models.InstanceTable,
}
}
type LockTable struct {
component.Base
}
var (
_ component.Table = (*LockTable)(nil)
)
func (*LockTable) TableInfo() component.TableInfo {
return component.TableInfo{
Model: reflectx.TypeOf[models.Lock](),
Name: models.LockTable,
}
}

56
internal/dis/component/sql/update.go
View file

@ -5,9 +5,9 @@ import (
"errors"
"fmt"
"io"
"reflect"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
"github.com/FAU-CDI/wisski-distillery/pkg/timex"
@ -76,56 +76,24 @@ func (sql *SQL) Update(ctx context.Context, progress io.Writer) error {
// wait for the database to come up
logging.LogMessage(progress, ctx, "Waiting for database update to be complete")
sql.WaitQueryTable(ctx)
tables := []struct {
name string
model any
table string
}{
{
"instance",
&models.Instance{},
models.InstanceTable,
},
{
"metadata",
&models.Metadatum{},
models.MetadataTable,
},
{
"snapshot",
&models.Export{},
models.ExportTable,
},
{
"lock",
&models.Lock{},
models.LockTable,
},
{
"users",
&models.User{},
models.UserTable,
},
{
"grant",
&models.Grant{},
models.GrantTable,
},
if err := sql.WaitQueryTable(ctx); err != nil {
return err
}
// migrate all of the tables!
return logging.LogOperation(func() error {
for _, table := range tables {
logging.LogMessage(progress, ctx, "migrating %q table", table.name)
db, err := sql.QueryTable(ctx, false, table.table)
for _, table := range sql.Dependencies.Tables {
info := table.TableInfo()
logging.LogMessage(progress, ctx, "migrating %q table", table.Name)
db, err := sql.queryTable(ctx, false, info.Name)
if err != nil {
return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
return errSQLUnableToMigrate.WithMessageF(table.Name, "unable to access table")
}
if err := db.AutoMigrate(table.model); err != nil {
return errSQLUnableToMigrate.WithMessageF(table.name, err)
tp := reflect.New(info.Model).Interface()
if err := db.AutoMigrate(tp); err != nil {
return errSQLUnableToMigrate.WithMessageF(table.Name, err)
}
}
return nil

18
internal/dis/component/table.go Normal file
View file

@ -0,0 +1,18 @@
package component
import (
"reflect"
)
// Table is a component that manages a provided sql table
type Table interface {
Component
// TableInfo returns information about a specific table
TableInfo() TableInfo
}
type TableInfo struct {
Model reflect.Type // model is the model this type manages
Name string
}