Move code into new component package

This commit cleans up the resources in the 'embed' package, and instead
moves them into subpackages of a new 'compose' package. This makes sure
that '.env' templates and docker compose contexts are located in the
same location.
This commit is contained in:
Tom Wiesing 2022-09-11 15:41:11 +02:00
parent 2ee90bf462
commit 7b2f79bea1
No known key found for this signature in database
44 changed files with 579 additions and 559 deletions

1
.gitignore vendored
View file

@ -1,5 +1,4 @@
/wdcli
/distillery/overrides.json
authorized_keys
.vagrant
.env

View file

@ -10,6 +10,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/execx"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
"github.com/FAU-CDI/wisski-distillery/internal/unpack"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/parser"
)
@ -143,7 +144,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
}
if err := logging.LogOperation(func() error {
return embed.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), func(dst, src string) {
return unpack.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), embed.ResourceEmbed, func(dst, src string) {
context.Printf("[copy] %s\n", dst)
})
}, context.IOStream, "Unpacking Runtime Components"); err != nil {

61
component/component.go Normal file
View file

@ -0,0 +1,61 @@
// Package component holds the main abstraction for components.
package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
// Component represents a logical subsystem of the distillery.
//
// By convention these are defined within their corresponding subpackage.
// This subpackage also contains all required resources.
// Furthermore, a component is typically instantiated using a call on the ["distillery.Distillery"] struct.
//
// Each Component should make use of [ComponentBase] for sane defaults.
//
// For example, the web.Web component lives in the web package and can be created like:
//
// var dis Distillery
// web := dis.Web()
type Component interface {
// Name returns the name of this component.
// It should correspond to the appropriate subpackage.
Name() string
// Path returns the path this component is installed at.
// By convention it is /var/www/deploy/core/${Name()}
Path() string
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call
Stack() stack.Installable
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
Context(parent stack.InstallationContext) stack.InstallationContext
}
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
Config *config.Config // Config is the configuration of the underlying distillery
}
// Path returns the path to this component
func (cb ComponentBase) Path() string {
return cb.Dir
}
// Context passes through the parent context
func (ComponentBase) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack stack.Installable) stack.Installable {
stack.Dir = cb.Dir
return stack
}

51
component/dis/dis.go Normal file
View file

@ -0,0 +1,51 @@
package dis
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/core"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
type Dis struct {
component.ComponentBase
// TODO: SQL Component
Executable string // path to the current executable
}
func (dis Dis) Name() string {
return "dis"
}
//go:embed all:stack dis.env
var resources embed.FS
func (dis Dis) Stack() stack.Installable {
return dis.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "dis.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": dis.Config.DefaultVirtualHost(),
"LETSENCRYPT_HOST": dis.Config.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
"CONFIG_PATH": dis.Config.ConfigPath,
"DEPLOY_ROOT": dis.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
},
CopyContextFiles: []string{core.Executable},
})
}
func (dis Dis) Context(parent stack.InstallationContext) stack.InstallationContext {
return stack.InstallationContext{
core.Executable: dis.Executable,
}
}

View file

@ -0,0 +1,110 @@
package resolver
import (
"embed"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/core"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
"github.com/tkw1536/goprogram/stream"
)
// TODO: Add a 'self-server' concept!
type Resolver struct {
component.ComponentBase
ConfigName string // the name to the config file
Executable string // path to the current executable
}
func (Resolver) Name() string {
return "resolver"
}
func (resolver Resolver) ConfigPath() string {
return filepath.Join(resolver.Dir, resolver.ConfigName)
}
//go:embed all:stack resolver.env
var resources embed.FS
func (resolver Resolver) Stack() stack.Installable {
return resolver.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "resolver.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": resolver.Config.DefaultVirtualHost(),
"LETSENCRYPT_HOST": resolver.Config.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": resolver.Config.CertbotEmail,
"CONFIG_PATH": resolver.Config.ConfigPath,
"DEPLOY_ROOT": resolver.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": resolver.Config.SelfOverridesFile,
"RESOLVER_CONFIG": resolver.ConfigPath(),
},
TouchFiles: []string{resolver.ConfigName},
CopyContextFiles: []string{core.Executable},
})
}
func (resolver Resolver) Context(parent stack.InstallationContext) stack.InstallationContext {
return stack.InstallationContext{
core.Executable: resolver.Executable,
}
}
func (resolver Resolver) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := resolver.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range resolver.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := resolver.ConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}

43
component/self/self.go Normal file
View file

@ -0,0 +1,43 @@
package self
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
type Self struct {
component.ComponentBase
}
func (Self) Name() string {
return "self"
}
//go:embed all:stack
//go:embed self.env
var resources embed.FS
func (self Self) Stack() stack.Installable {
// TODO: Move me into config!
TARGET := "https://github.com/FAU-CDI/wisski-distillery"
if self.Config.SelfRedirect != nil { // TODO: move to config!
TARGET = self.Config.SelfRedirect.String()
}
return self.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "self.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": self.Config.DefaultVirtualHost(),
"LETSENCRYPT_HOST": self.Config.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": self.Config.CertbotEmail,
"TARGET": TARGET,
"OVERRIDES_FILE": self.Config.SelfOverridesFile,
},
})
}

View file

@ -1,69 +1,24 @@
package env
package sql
import (
"errors"
"fmt"
"io"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/FAU-CDI/wisski-distillery/internal/sqle"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
"github.com/FAU-CDI/wisski-distillery/internal/wait"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// SQLComponent represents the 'sql' layer belonging to a distillery
type SQLComponent struct {
ServerURL string
PollInterval time.Duration // Duration to wait for during wait
dis *Distillery
}
// SSH returns the SSHComponent belonging to this distillery
func (dis *Distillery) SQL() SQLComponent {
return SQLComponent{
ServerURL: dis.Upstream.SQL,
PollInterval: time.Second,
dis: dis,
}
}
func (SQLComponent) Name() string {
return "sql"
}
func (SQLComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
// Stack returns the docker stack that handles the sql database.
func (sql SQLComponent) Stack() stack.Installable {
return sql.dis.makeComponentStack(sql, stack.Installable{
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
"data",
},
})
}
// SQLStackPath returns the path the SQLStack() lives at.
func (sql SQLComponent) Path() string {
return sql.Stack().Dir
}
// sqlOpen opens a new sql connection to the provided database using the administrative credentials
func (sql SQLComponent) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
cfg := mysql.Config{
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.dis.Config.MysqlAdminUser, sql.dis.Config.MysqlAdminPassword, sql.ServerURL, database),
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
DefaultStringSize: 256,
}
@ -82,7 +37,7 @@ func (sql SQLComponent) openDatabase(database string, config *gorm.Config) (*gor
}
// OpenBookkeeping opens a connection to the bookkeeping database
func (sql SQLComponent) OpenBookkeeping(silent bool) (*gorm.DB, error) {
func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
config := &gorm.Config{}
if silent {
@ -90,13 +45,13 @@ func (sql SQLComponent) OpenBookkeeping(silent bool) (*gorm.DB, error) {
}
// open the database
db, err := sql.openDatabase(sql.dis.Config.DistilleryBookkeepingDatabase, config)
db, err := sql.openDatabase(sql.Config.DistilleryBookkeepingDatabase, config)
if err != nil {
return nil, err
}
// load the table
table := db.Table(sql.dis.Config.DistilleryBookkeepingTable)
table := db.Table(sql.Config.DistilleryBookkeepingTable)
if table.Error != nil {
return nil, err
}
@ -107,7 +62,7 @@ func (sql SQLComponent) OpenBookkeeping(silent bool) (*gorm.DB, error) {
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
// Backup makes a backup of the sql database into dest.
func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database string) error {
func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
@ -121,7 +76,7 @@ func (sql SQLComponent) Backup(io stream.IOStream, dest io.Writer, database stri
}
// BackupAll makes a backup of all sql databases
func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error {
func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
@ -135,37 +90,37 @@ func (sql SQLComponent) BackupAll(io stream.IOStream, dest io.Writer) error {
}
// OpenShell executes a mysql shell command
func (sql SQLComponent) OpenShell(io stream.IOStream, argv ...string) (int, error) {
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)
}
// WaitShell waits for the sql database to be reachable via a docker-compose shell
func (sql SQLComponent) WaitShell() error {
func (sql SQL) WaitShell() error {
n := stream.FromNil()
return wait.Wait(func() bool {
code, err := sql.OpenShell(n, "-e", "show databases;")
return err == nil && code == 0
}, sql.PollInterval, sql.dis.Context())
}, sql.PollInterval, sql.PollContext)
}
// Wait waits for a connection to the bookkeeping table to suceed
func (sql SQLComponent) Wait() error {
func (sql SQL) Wait() error {
return wait.Wait(func() bool {
_, err := sql.OpenBookkeeping(true)
return err == nil
}, sql.PollInterval, sql.dis.Context())
}, sql.PollInterval, sql.PollContext)
}
var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
func (sql SQLComponent) Query(query string, args ...interface{}) bool {
func (sql SQL) Query(query string, args ...interface{}) bool {
raw := sqle.Format(query, args...)
code, err := sql.OpenShell(stream.FromNil(), "-e", raw)
return err == nil && code == 0
}
// SQLProvision provisions a new sql database and user
func (sql SQLComponent) Provision(name, user, password string) error {
func (sql SQL) Provision(name, user, password string) error {
// wait for the database
if err := sql.WaitShell(); err != nil {
return err
@ -188,7 +143,7 @@ func (sql SQLComponent) Provision(name, user, password string) error {
var errSQLPurgeUser = errors.New("unable to delete user")
// SQLPurgeUser deletes the specified user from the database
func (sql SQLComponent) PurgeUser(user string) error {
func (sql SQL) PurgeUser(user string) error {
if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
return errSQLPurgeUser
}
@ -199,7 +154,7 @@ func (sql SQLComponent) PurgeUser(user string) error {
var errSQLPurgeDB = errors.New("unable to drop database")
// SQLPurgeDatabase deletes the specified db from the database
func (sql SQLComponent) PurgeDatabase(db string) error {
func (sql SQL) PurgeDatabase(db string) error {
if !sqle.IsSafeDatabaseName(db) {
return errSQLPurgeDB
}
@ -214,7 +169,7 @@ var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe na
var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
func (sql SQLComponent) Bootstrap(io stream.IOStream) error {
func (sql SQL) Bootstrap(io stream.IOStream) error {
if err := sql.WaitShell(); err != nil {
return err
}
@ -222,8 +177,8 @@ func (sql SQLComponent) Bootstrap(io stream.IOStream) error {
// create the admin user
logging.LogMessage(io, "Creating administrative user")
{
username := sql.dis.Config.MysqlAdminUser
password := sql.dis.Config.MysqlAdminPassword
username := sql.Config.MysqlAdminUser
password := sql.Config.MysqlAdminPassword
if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
return errSQLUnableToCreateUser
}
@ -232,10 +187,10 @@ func (sql SQLComponent) Bootstrap(io stream.IOStream) error {
// create the admin user
logging.LogMessage(io, "Creating sql database")
{
if !sqle.IsSafeDatabaseName(sql.dis.Config.DistilleryBookkeepingDatabase) {
if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) {
return errSQLUnsafeDatabaseName
}
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.dis.Config.DistilleryBookkeepingDatabase)
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase)
if !sql.Query(createDBSQL) {
return errSQLUnableToCreate
}

39
component/sql/sql.go Normal file
View file

@ -0,0 +1,39 @@
package sql
import (
"context"
"embed"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
type SQL struct {
component.ComponentBase
ServerURL string // upstream server url
PollContext context.Context // context to abort polling with
PollInterval time.Duration // duration to wait for during wait
}
func (SQL) Name() string {
return "sql"
}
//go:embed all:stack
var resources embed.FS
func (ssh SQL) Stack() stack.Installable {
return ssh.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
"data",
},
})
}

26
component/ssh/ssh.go Normal file
View file

@ -0,0 +1,26 @@
package ssh
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
type SSH struct {
component.ComponentBase
}
func (SSH) Name() string {
return "ssh"
}
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() stack.Installable {
return ssh.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
})
}

View file

@ -1,4 +1,4 @@
package env
package triplestore
import (
"bytes"
@ -10,12 +10,10 @@ import (
"net/http"
"os"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/embed"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/logging"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
"github.com/FAU-CDI/wisski-distillery/internal/unpack"
"github.com/FAU-CDI/wisski-distillery/internal/wait"
"github.com/pkg/errors"
@ -23,50 +21,6 @@ import (
"github.com/tkw1536/goprogram/stream"
)
// TriplestoreComponent represents the triplestore belonging to a distillery
type TriplestoreComponent struct {
BaseURL string // the base url of the api
PollInterval time.Duration // duration to wait during wait!
dis *Distillery
}
// Triplestore returns the TriplestoreComponent belonging to this distillery
func (dis *Distillery) Triplestore() TriplestoreComponent {
return TriplestoreComponent{
BaseURL: "http://" + dis.Upstream.Triplestore,
PollInterval: time.Second,
dis: dis,
}
}
func (TriplestoreComponent) Name() string {
return "triplestore"
}
func (TriplestoreComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
// Stack returns the installable Triplestore stack
func (ts TriplestoreComponent) Stack() stack.Installable {
return ts.dis.makeComponentStack(ts, stack.Installable{
CopyContextFiles: []string{"graphdb.zip"},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),
filepath.Join("data", "logs"),
},
})
}
func (ts TriplestoreComponent) Path() string {
return ts.Stack().Dir
}
type TriplestoreUserPayload struct {
Password string `json:"password"`
AppSettings TriplestoreUserAppSettings `json:"appSettings"`
@ -84,7 +38,7 @@ type TriplestoreUserAppSettings struct {
//
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
// When bodyName is empty, simply marshal body as application/json
func (ts TriplestoreComponent) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
var reader io.Reader
var contentType string
@ -126,7 +80,7 @@ func (ts TriplestoreComponent) OpenRaw(method, url string, body interface{}, bod
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
req.SetBasicAuth(ts.dis.Config.TriplestoreAdminUser, ts.dis.Config.TriplestoreAdminPassword)
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
// and send it
return http.DefaultClient.Do(req)
@ -134,7 +88,7 @@ func (ts TriplestoreComponent) OpenRaw(method, url string, body interface{}, bod
// Wait waits for the connection to the Triplestore to succeed.
// This is achieved using a polling strategy.
func (ts TriplestoreComponent) Wait() error {
func (ts Triplestore) Wait() error {
return wait.Wait(func() bool {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
if err != nil {
@ -142,7 +96,7 @@ func (ts TriplestoreComponent) Wait() error {
}
defer res.Body.Close()
return true
}, ts.PollInterval, ts.dis.Context())
}, ts.PollInterval, ts.PollContext)
}
var errTripleStoreFailedRepository = exit.Error{
@ -150,7 +104,7 @@ var errTripleStoreFailedRepository = exit.Error{
ExitCode: exit.ExitGeneric,
}
func (ts TriplestoreComponent) Provision(name, domain, user, password string) error {
func (ts Triplestore) Provision(name, domain, user, password string) error {
if err := ts.Wait(); err != nil {
return err
}
@ -210,7 +164,7 @@ func (ts TriplestoreComponent) Provision(name, domain, user, password string) er
}
// TriplestorePurgeUser deletes the specified user from the triplestore
func (ts TriplestoreComponent) PurgeUser(user string) error {
func (ts Triplestore) PurgeUser(user string) error {
res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
if err != nil {
return err
@ -222,7 +176,7 @@ func (ts TriplestoreComponent) PurgeUser(user string) error {
}
// TriplestorePurgeRepo deletes the specified repo from the triplestore
func (ts TriplestoreComponent) PurgeRepo(repo string) error {
func (ts Triplestore) PurgeRepo(repo string) error {
res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
if err != nil {
return err
@ -236,7 +190,7 @@ func (ts TriplestoreComponent) PurgeRepo(repo string) error {
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
// TriplestoreBackup backs up the repository named repo into the writer dst.
func (ts TriplestoreComponent) Backup(dst io.Writer, repo string) (int64, error) {
func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
if err != nil {
return 0, err
@ -260,7 +214,7 @@ type Repository struct {
Local bool `json:"local"`
}
func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error) {
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
if err != nil {
return nil, err
@ -272,7 +226,7 @@ func (ts TriplestoreComponent) listRepositories() (repos []Repository, err error
}
// TriplestoreBackup backs up every graphdb instance into dst
func (ts TriplestoreComponent) BackupAll(dst string) error {
func (ts Triplestore) BackupAll(dst string) error {
// list all the repositories
repos, err := ts.listRepositories()
if err != nil {
@ -306,7 +260,7 @@ func (ts TriplestoreComponent) BackupAll(dst string) error {
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {
func (ts Triplestore) Bootstrap(io stream.IOStream) error {
logging.LogMessage(io, "Waiting for Triplestore")
if err := ts.Wait(); err != nil {
return err
@ -314,8 +268,8 @@ func (ts TriplestoreComponent) Bootstrap(io stream.IOStream) error {
logging.LogMessage(io, "Resetting admin user password")
{
res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.dis.Config.TriplestoreAdminUser, TriplestoreUserPayload{
Password: ts.dis.Config.TriplestoreAdminPassword,
res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
Password: ts.Config.TriplestoreAdminPassword,
AppSettings: TriplestoreUserAppSettings{
DefaultInference: true,
DefaultVisGraphSchema: true,

View file

@ -0,0 +1,44 @@
package triplestore
import (
"context"
"embed"
"io/fs"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
type Triplestore struct {
component.ComponentBase
BaseURL string // upstream server url
PollContext context.Context // context to abort polling with
PollInterval time.Duration // duration to wait for during wait
}
func (Triplestore) Name() string {
return "triplestore"
}
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() stack.Installable {
return ts.ComponentBase.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),
filepath.Join("data", "logs"),
},
})
}

39
component/web/web.go Normal file
View file

@ -0,0 +1,39 @@
package web
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
// Web implements the web component
type Web struct {
component.ComponentBase
}
func (Web) Name() string {
return "web"
}
//go:embed all:stack
//go:embed web.env
var resources embed.FS
func (web Web) Stack() stack.Installable {
HTTPS_METHOD := "nohttp"
if web.Config.HTTPSEnabled() {
HTTPS_METHOD = "redirect"
}
return web.MakeStack(stack.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "web.env",
EnvContext: map[string]string{
"DEFAULT_HOST": web.Config.DefaultDomain,
"HTTPS_METHOD": HTTPS_METHOD,
},
})
}

View file

@ -1,118 +0,0 @@
// Package embed contains embedded resources
package embed
import (
"embed"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// ResourceEmbed contains all the resources required by the WissKI-Distillery package.
//go:embed all:resources
var ResourceEmbed embed.FS
// InstallResource install a resource src into dest.
// When it encounters a directory, recursively installs the directory is called.
// For each installation item, onInstallFile is called, unless onInstallFile is nil.
//
// If src points to a file, dst must either be an existing file, or not exist.
// If src points to a directory, dst must either be an existing directory, or not exist.
func InstallResource(dst, src string, onInstallFile func(dst, src string)) error {
return installFile(dst, ResourceEmbed, src, onInstallFile)
}
var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory")
var errExpectedDirectoryButGotFile = errors.New("Expected a directory, but got a file")
func installFile(dst string, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// call the on-install file path
if onInstallFile != nil {
onInstallFile(dst, src)
}
// open the source file!
srcFile, err := fsys.Open(src)
if err != nil {
return errors.Wrapf(err, "Error opening source file %s", src)
}
defer srcFile.Close()
// stat the source file to install
srcStat, srcErr := srcFile.Stat()
if srcErr != nil {
return errors.Wrapf(srcErr, "Error calling stat on source %s", src)
}
// if it is a directory, we should recurse!
if srcStat.IsDir() {
return installDir(dst, srcStat, srcFile, fsys, src, onInstallFile)
}
// determine if we need to create the destination file, or if it already exists
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case dstStat.IsDir():
return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
}
// Open the file
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcStat.Mode())
if err != nil {
return errors.Wrapf(err, "Error opening destination %s", dst)
}
defer dstFile.Close()
// copy over the content
_, err = io.Copy(dstFile, srcFile)
return errors.Wrapf(err, "Error writing to destination %s", dst)
}
func installDir(dst string, srcStat fs.FileInfo, srcFile fs.File, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// make sure it is a directory!
dir, ok := srcFile.(fs.ReadDirFile)
if !ok {
return errExpectedDirectoryButGotFile
}
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcStat.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case !dstStat.IsDir():
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
case dstErr == nil:
}
// read the directory
entries, err := dir.ReadDir(-1)
if err != nil {
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
}
// iterate over all the children
for _, entry := range entries {
if err := func(dst, src string) error {
return installFile(dst, fsys, src, onInstallFile)
}(
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
); err != nil {
return err
}
}
return nil
}

10
embed/legacy.go Normal file
View file

@ -0,0 +1,10 @@
// Package embed contains embedded resources
package embed
import (
"embed"
)
// ResourceEmbed contains all the resources required by the WissKI-Distillery package.
//go:embed all:resources
var ResourceEmbed embed.FS

View file

@ -1 +0,0 @@
package embed

78
env/component.go vendored
View file

@ -2,15 +2,27 @@ package env
import (
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/component"
"github.com/FAU-CDI/wisski-distillery/component/dis"
"github.com/FAU-CDI/wisski-distillery/component/resolver"
"github.com/FAU-CDI/wisski-distillery/component/self"
"github.com/FAU-CDI/wisski-distillery/component/sql"
"github.com/FAU-CDI/wisski-distillery/component/ssh"
"github.com/FAU-CDI/wisski-distillery/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/component/web"
"github.com/FAU-CDI/wisski-distillery/embed"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
// TODO: Remove me when migration is complete
type Component = component.Component
// TODO: Move everything into specific subpackages
// Stacks returns the Stacks of this distillery
func (dis *Distillery) Components() []Component {
func (dis *Distillery) Components() []component.Component {
// TODO: Do we want to cache these components?
return []Component{
dis.Web(),
@ -23,17 +35,69 @@ func (dis *Distillery) Components() []Component {
}
}
// Component represents a component of the distillery
type Component interface {
Name() string // Name is the name of this component
// Web returns the web component belonging to this distillery
func (dis *Distillery) Web() (web web.Web) {
dis.makeComponent(web, &web.ComponentBase)
return
}
Stack() stack.Installable // Stack returns the installable stack representing this component
Context(parent stack.InstallationContext) stack.InstallationContext // context for installation
// Self returns the self component belonging to this distillery
func (dis *Distillery) Self() (self self.Self) {
dis.makeComponent(self, &self.ComponentBase)
return
}
Path() string // Path returns the path to this component
// Resolver returns the resolver component belonging to this distillery
func (dis *Distillery) Resolver() (resolver resolver.Resolver) {
resolver.ConfigName = "prefix.cfg" // TODO: Move into core?
resolver.Executable = dis.CurrentExecutable()
dis.makeComponent(resolver, &resolver.ComponentBase)
return
}
// Dis returns the dis component belonging to this distillery
func (dis *Distillery) Dis() (ddis dis.Dis) {
ddis.Executable = dis.CurrentExecutable()
dis.makeComponent(ddis, &ddis.ComponentBase)
return
}
// SSH returns the SSH component belonging to this distillery
func (dis *Distillery) SSH() (ssh ssh.SSH) {
dis.makeComponent(ssh, &ssh.ComponentBase)
return
}
// SQL returns the SQL component belonging to this distillery
func (dis *Distillery) SQL() (sql sql.SQL) {
sql.ServerURL = dis.Upstream.SQL
sql.PollContext = dis.Context()
sql.PollInterval = time.Second
dis.makeComponent(sql, &sql.ComponentBase)
return
}
// Triplestore returns the TriplestoreComponent belonging to this distillery
func (dis *Distillery) Triplestore() (ts triplestore.Triplestore) {
ts.BaseURL = "http://" + dis.Upstream.Triplestore
ts.PollContext = dis.Context()
ts.PollInterval = time.Second
dis.makeComponent(ts, &ts.ComponentBase)
return
}
// makeComponent updates the baseComponent belonging to component
func (dis *Distillery) makeComponent(component component.Component, base *component.ComponentBase) {
base.Dir = dis.getComponentPath(component)
base.Config = dis.Config
}
// asCoreStack treats the provided stack as a core component of this distillery.
// TODO: this should no longer be used
func (dis *Distillery) makeComponentStack(component Component, stack stack.Installable) stack.Installable {
stack.Dir = dis.getComponentPath(component)

47
env/component_dis.go vendored
View file

@ -1,47 +0,0 @@
package env
import (
"github.com/FAU-CDI/wisski-distillery/core"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
)
// DisComponent represents the 'dis' layer belonging to a distillery
type DisComponent struct {
dis *Distillery
}
// Dis returns the DisComponent belonging to this distillery
func (dis *Distillery) Dis() DisComponent {
return DisComponent{dis: dis}
}
func (DisComponent) Name() string {
return "dis"
}
func (dis DisComponent) Stack() stack.Installable {
return dis.dis.makeComponentStack(dis, stack.Installable{
EnvContext: map[string]string{
"VIRTUAL_HOST": dis.dis.DefaultVirtualHost(),
"LETSENCRYPT_HOST": dis.dis.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": dis.dis.Config.CertbotEmail,
"CONFIG_PATH": dis.dis.Config.ConfigPath,
"DEPLOY_ROOT": dis.dis.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.dis.Config.SelfOverridesFile,
},
CopyContextFiles: []string{core.Executable},
})
}
func (dis DisComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return stack.InstallationContext{
core.Executable: dis.dis.CurrentExecutable(),
}
}
func (dis DisComponent) Path() string {
return dis.Stack().Dir
}

View file

@ -1,112 +0,0 @@
package env
import (
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/FAU-CDI/wisski-distillery/core"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
"github.com/tkw1536/goprogram/stream"
)
// ResolverComponent represents the 'resolver' layer belonging to a distillery
type ResolverComponent struct {
ConfigName string // Filename of the configuration file
dis *Distillery
}
// Resolver returns the ResolverComponent belonging to this distillery
func (dis *Distillery) Resolver() ResolverComponent {
return ResolverComponent{
ConfigName: "prefix.cfg",
dis: dis,
}
}
func (ResolverComponent) Name() string {
return "resolver"
}
func (resolver ResolverComponent) Stack() stack.Installable {
return resolver.dis.makeComponentStack(resolver, stack.Installable{
EnvContext: map[string]string{
"VIRTUAL_HOST": resolver.dis.DefaultVirtualHost(),
"LETSENCRYPT_HOST": resolver.dis.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": resolver.dis.Config.CertbotEmail,
"CONFIG_PATH": resolver.dis.Config.ConfigPath,
"DEPLOY_ROOT": resolver.dis.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": resolver.dis.Config.SelfOverridesFile,
"RESOLVER_CONFIG": resolver.ConfigPath(),
},
TouchFiles: []string{resolver.ConfigName},
CopyContextFiles: []string{core.Executable},
})
}
func (resolver ResolverComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return stack.InstallationContext{
core.Executable: resolver.dis.CurrentExecutable(),
}
}
func (resolver ResolverComponent) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := resolver.dis.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range resolver.dis.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := resolver.ConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}
func (resolver ResolverComponent) Path() string {
return resolver.dis.getComponentPath(resolver)
}
func (resolver ResolverComponent) ConfigPath() string {
return filepath.Join(resolver.Path(), resolver.ConfigName)
}

42
env/component_self.go vendored
View file

@ -1,42 +0,0 @@
package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack"
// SelfComponent represents the 'self' layer belonging to a distillery
type SelfComponent struct {
dis *Distillery
}
// Self returns the SelfComponent belonging to this distillery
func (dis *Distillery) Self() SelfComponent {
return SelfComponent{dis: dis}
}
func (SelfComponent) Name() string {
return "self"
}
func (SelfComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
func (sc SelfComponent) Stack() stack.Installable {
TARGET := "https://github.com/FAU-CDI/wisski-distillery"
if sc.dis.Config.SelfRedirect != nil {
TARGET = sc.dis.Config.SelfRedirect.String()
}
return sc.dis.makeComponentStack(sc, stack.Installable{
EnvContext: map[string]string{
"VIRTUAL_HOST": sc.dis.DefaultVirtualHost(),
"LETSENCRYPT_HOST": sc.dis.DefaultLetsencryptHost(),
"LETSENCRYPT_EMAIL": sc.dis.Config.CertbotEmail,
"TARGET": TARGET,
"OVERRIDES_FILE": sc.dis.Config.SelfOverridesFile,
},
})
}
func (sc SelfComponent) Path() string {
return sc.Stack().Dir
}

29
env/component_ssh.go vendored
View file

@ -1,29 +0,0 @@
package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack"
// SSHComponent represents the 'ssh' layer belonging to a distillery
type SSHComponent struct {
dis *Distillery
}
// SSH returns the SSHComponent belonging to this distillery
func (dis *Distillery) SSH() SSHComponent {
return SSHComponent{dis: dis}
}
func (SSHComponent) Name() string {
return "ssh"
}
func (ssh SSHComponent) Stack() stack.Installable {
return ssh.dis.makeComponentStack(ssh, stack.Installable{})
}
func (SSHComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
func (ssh SSHComponent) Path() string {
return ssh.Stack().Dir
}

39
env/component_web.go vendored
View file

@ -1,39 +0,0 @@
package env
import "github.com/FAU-CDI/wisski-distillery/internal/stack"
// WebComponent represents the 'web' layer belonging to a distillery
type WebComponent struct {
dis *Distillery
}
// Web returns the WebComponent belonging to this distillery
func (dis *Distillery) Web() WebComponent {
return WebComponent{dis: dis}
}
func (WebComponent) Name() string {
return "web"
}
func (web WebComponent) Stack() stack.Installable {
HTTPS_METHOD := "nohttp"
if web.dis.HTTPSEnabled() {
HTTPS_METHOD = "redirect"
}
return web.dis.makeComponentStack(web, stack.Installable{
EnvContext: map[string]string{
"DEFAULT_HOST": web.dis.Config.DefaultDomain,
"HTTPS_METHOD": HTTPS_METHOD,
},
})
}
func (WebComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
return parent
}
func (web WebComponent) Path() string {
return web.Stack().Dir
}

21
env/distillery.go vendored
View file

@ -3,7 +3,6 @@ package env
import (
"context"
"os"
"strings"
"github.com/FAU-CDI/wisski-distillery/core"
"github.com/FAU-CDI/wisski-distillery/internal/config"
@ -22,26 +21,6 @@ type Upstream struct {
Triplestore string
}
func (dis Distillery) HTTPSEnabled() bool {
return dis.Config.CertbotEmail != ""
}
// Returns the default virtual host
func (dis Distillery) DefaultVirtualHost() string {
VIRTUAL_HOST := dis.Config.DefaultDomain
if len(dis.Config.SelfExtraDomains) > 0 {
VIRTUAL_HOST += "," + strings.Join(dis.Config.SelfExtraDomains, ",")
}
return VIRTUAL_HOST
}
func (dis Distillery) DefaultLetsencryptHost() string {
if !dis.HTTPSEnabled() {
return ""
}
return dis.DefaultVirtualHost()
}
// Context returns a new Context belonging to this distillery
func (dis Distillery) Context() context.Context {
return context.Background()

8
env/instances.go vendored
View file

@ -11,6 +11,7 @@ import (
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/embed"
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/stack"
@ -201,9 +202,9 @@ func (instance Instance) Domain() string {
}
// IfHttps returns value if the distillery has https enabled, the empty string otherwise
// TODO: Fix this to be in a proper place
// TODO: Fix this into config!
func (dis *Distillery) IfHttps(value string) string {
if !dis.HTTPSEnabled() {
if !dis.Config.HTTPSEnabled() {
return ""
}
return value
@ -218,7 +219,7 @@ func (instance Instance) URL() *url.URL {
}
// use http or https scheme depending on if the distillery has it enabled
if instance.dis.HTTPSEnabled() {
if instance.dis.Config.HTTPSEnabled() {
url.Scheme = "https"
} else {
url.Scheme = "http"
@ -233,6 +234,7 @@ func (instance Instance) Stack() stack.Installable {
Stack: stack.Stack{
Dir: instance.FilesystemBase,
},
Resources: embed.ResourceEmbed, // TODO: Move this over
ContextPath: filepath.Join("resources", "compose", "barrel"),
EnvPath: filepath.Join("resources", "templates", "docker-env", "barrel"),

2
env/server.go vendored
View file

@ -5,6 +5,8 @@ import (
"net/http"
)
// TODO: Move this into dis!
// Server represents a server for this distillery
type Server struct {
dis *Distillery

View file

@ -0,0 +1,25 @@
package config
import "strings"
// This file contains derived configuration values
func (cfg Config) HTTPSEnabled() bool {
return cfg.CertbotEmail != ""
}
// Returns the default virtual host
func (cfg Config) DefaultVirtualHost() string {
VIRTUAL_HOST := cfg.DefaultDomain
if len(cfg.SelfExtraDomains) > 0 {
VIRTUAL_HOST += "," + strings.Join(cfg.SelfExtraDomains, ",")
}
return VIRTUAL_HOST
}
func (cfg Config) DefaultLetsencryptHost() string {
if !cfg.HTTPSEnabled() {
return ""
}
return cfg.DefaultVirtualHost()
}

View file

@ -5,13 +5,14 @@ import (
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/embed"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/unpack"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
// TODO: Move this package into components
// Installable represents a Stack that can be automatically installed from a set of resources
// See the [Install] method.
type Installable struct {
@ -42,15 +43,18 @@ type InstallationContext map[string]string
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
// setup the base files
if err := embed.InstallResource(
is.Dir,
is.ContextPath,
func(dst, src string) {
io.Printf("[install] %s\n", dst)
},
); err != nil {
return err
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallResource(
is.Dir,
is.ContextPath,
is.Resources,
func(dst, src string) {
io.Printf("[install] %s\n", dst)
},
); err != nil {
return err
}
}
// configure .env