Refactor Package structure

This commit cleans up the package structure, to make two new top-level
packages `internal` (for internal-use packages) and `pkg` (for general
shared utility code).
This commit is contained in:
Tom Wiesing 2022-09-12 14:46:18 +02:00
parent 487ce09979
commit a360324f62
No known key found for this signature in database
124 changed files with 97 additions and 101 deletions

View file

@ -1,68 +0,0 @@
// Package bookkeeping implements reading and writing from the bookkeeping table
package bookkeeping
import (
"database/sql/driver"
"errors"
"time"
)
// Instance is a WissKI Instance inside the bookkeeping table.
// It does not represent a running instance; it does not perform any validation.
type Instance struct {
// NOTE: Modifying this struct requires a database migration.
// This should nnever be done unless you know what you're doing.
// Primary key for the instance
Pk uint `gorm:"column:pk;primaryKey"`
// time the instance was created
Created time.Time `gorm:"column:created;autoCreateTime"`
// slug of the system
Slug string `gorm:"column:slug;not null;unique"`
// email address of the system owner (if any)
OwnerEmail string `gorm:"column:owner_email;type:varchar(320)"`
// should we automatically enable updates for the system?
AutoBlindUpdateEnabled SQLBit1 `gorm:"column:auto_blind_update_enabled;default:1"`
// The filesystem path the system can be found under
FilesystemBase string `gorm:"column:filesystem_base;not null"`
// SQL Database credentials for the system
SqlDatabase string `gorm:"column:sql_database;not null"`
SqlUser string `gorm:"column:sql_user;not null"`
SqlPassword string `gorm:"column:sql_password;not null"`
// GraphDB Repository
GraphDBRepository string `gorm:"column:graphdb_repository;not null"`
GraphDBUser string `gorm:"column:graphdb_user;not null"`
GraphDBPassword string `gorm:"column:graphdb_password;not null"`
}
func (i Instance) IsBlindUpdateEnabled() bool {
return bool(i.AutoBlindUpdateEnabled)
}
// SQLBit1 implements a boolean as a BIT(1)
type SQLBit1 bool
func (sb SQLBit1) Value() (driver.Value, error) {
if sb {
return []byte{1}, nil
} else {
return []byte{0}, nil
}
}
var errBadBool = errors.New("SQLBit1: Database does not contain Bit(1)")
func (sb *SQLBit1) Scan(src interface{}) error {
if bytes, ok := src.([]byte); ok && len(bytes) == 1 {
*sb = bytes[0] == 1
return nil
}
return errBadBool
}

View file

@ -0,0 +1,69 @@
// Package component holds the main abstraction for components.
package component
import (
"github.com/FAU-CDI/wisski-distillery/internal/config"
)
// Component represents a logical subsystem of the distillery.
//
// By convention these are defined within their corresponding subpackage.
// This subpackage also contains all required resources.
// Furthermore, a component is typically instantiated using a call on the ["distillery.Distillery"] struct.
//
// Each Component should make use of [ComponentBase] for sane defaults.
//
// For example, the web.Web component lives in the web package and can be created like:
//
// var dis Distillery
// web := dis.Web()
type Component interface {
// Name returns the name of this component.
// It should correspond to the appropriate subpackage.
Name() string
// Path returns the path this component is installed at.
// By convention it is /var/www/deploy/internal/core/${Name()}
Path() string
// Stack can be used to gain access to the "docker compose" stack.
//
// This should internally call
Stack() Installable
// Context returns a new InstallationContext to be used during installation from the command line.
// Typically this should just pass through the parent, but might perform other tasks.
Context(parent InstallationContext) InstallationContext
// Base() returns a reference to a base component
// This is implemented by an embedding on ComponentBase
Base() *ComponentBase
}
// ComponentBase implements base functionality for a component
type ComponentBase struct {
Dir string // Dir is the directory this component lives in
Config *config.Config // Config is the configuration of the underlying distillery
}
// Base returns a reference to the ComponentBase
func (cb *ComponentBase) Base() *ComponentBase {
return cb
}
// Path returns the path to this component
func (cb ComponentBase) Path() string {
return cb.Dir
}
// Context passes through the parent context
func (ComponentBase) Context(parent InstallationContext) InstallationContext {
return parent
}
// MakeStack registers the Installable as a stack
func (cb ComponentBase) MakeStack(stack Installable) Installable {
stack.Dir = cb.Dir
return stack
}

View file

@ -0,0 +1,9 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}

View file

@ -0,0 +1,48 @@
package dis
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
type Dis struct {
component.ComponentBase
Executable string // path to the current executable
}
func (dis Dis) Name() string {
return "dis"
}
//go:embed all:stack dis.env
var resources embed.FS
func (dis Dis) Stack() component.Installable {
return dis.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "dis.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": dis.Config.DefaultHost(),
"LETSENCRYPT_HOST": dis.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
"CONFIG_PATH": dis.Config.ConfigPath,
"DEPLOY_ROOT": dis.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
},
CopyContextFiles: []string{core.Executable},
})
}
func (dis Dis) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: dis.Executable,
}
}

View file

@ -0,0 +1,5 @@
FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","dis_server","--bind","0.0.0.0:8888"]

View file

@ -0,0 +1,28 @@
version: "3.7"
services:
wdresolve:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /dis/
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,118 @@
package component
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
// TODO: Move this package into components
// Installable represents a Stack that can be automatically installed from a set of resources
// See the [Install] method.
type Installable struct {
Stack
// Installable enabled installing several resources from a (potentially embedded) filesystem.
//
// The Resources holds these, with appropriate resources specified below.
// These all refer to paths within the Resource filesystem.
Resources fs.FS
ContextPath string // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate].
EnvContext map[string]string // context when instantiating the '.env' template
CopyContextFiles []string // Files to copy from the installation context
MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
MakeDirs []string // directories to ensure that exist
TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
}
// InstallationContext is a context to install data in
type InstallationContext map[string]string
// Install installs or updates this stack into the directory specified by stack.Stack().
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
if is.ContextPath != "" {
// setup the base files
if err := unpack.InstallDir(
is.Dir,
is.ContextPath,
is.Resources,
func(dst, src string) {
io.Printf("[install] %s\n", dst)
},
); err != nil {
return err
}
}
// configure .env
envDest := filepath.Join(is.Dir, ".env")
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
if err := unpack.InstallTemplate(
envDest,
is.EnvContext,
is.EnvPath,
is.Resources,
); err != nil {
return err
}
}
// make sure that certain dirs exist
for _, name := range is.MakeDirs {
// find the destination!
dst := filepath.Join(is.Dir, name)
io.Printf("[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = fs.ModeDir
}
if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
return err
}
}
// copy files from the context!
for _, name := range is.CopyContextFiles {
// find the source!
src, ok := context[name]
if !ok {
return errors.Errorf("Missing file from context: %s", src)
}
// find the destination!
dst := filepath.Join(is.Dir, name)
// copy over file from context
io.Printf("[copy] %s (from %s)\n", dst, src)
if err := fsx.CopyFile(dst, src); err != nil {
return errors.Wrapf(err, "Unable to copy file %s", src)
}
}
// make sure that certain files exist
for _, name := range is.TouchFiles {
// find the destination!
dst := filepath.Join(is.Dir, name)
io.Printf("[touch] %s\n", dst)
if err := fsx.Touch(dst); err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1,10 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
RESOLVER_CONFIG=${RESOLVER_CONFIG}

View file

@ -0,0 +1,109 @@
package resolver
import (
"embed"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/tkw1536/goprogram/stream"
)
// TODO: Add a 'self-server' concept!
type Resolver struct {
component.ComponentBase
ConfigName string // the name to the config file
Executable string // path to the current executable
}
func (Resolver) Name() string {
return "resolver"
}
func (resolver Resolver) ConfigPath() string {
return filepath.Join(resolver.Dir, resolver.ConfigName)
}
//go:embed all:stack resolver.env
var resources embed.FS
func (resolver Resolver) Stack() component.Installable {
return resolver.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "resolver.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": resolver.Config.DefaultHost(),
"LETSENCRYPT_HOST": resolver.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": resolver.Config.CertbotEmail,
"CONFIG_PATH": resolver.Config.ConfigPath,
"DEPLOY_ROOT": resolver.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": resolver.Config.SelfOverridesFile,
"RESOLVER_CONFIG": resolver.ConfigPath(),
},
TouchFiles: []string{resolver.ConfigName},
CopyContextFiles: []string{core.Executable},
})
}
func (resolver Resolver) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: resolver.Executable,
}
}
func (resolver Resolver) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := resolver.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range resolver.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := resolver.ConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}

View file

@ -0,0 +1,5 @@
FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]

View file

@ -0,0 +1,29 @@
version: "3.7"
services:
wdresolve:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /go/
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
- "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,7 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
TARGET=${TARGET}
OVERRIDES_FILE=${OVERRIDES_FILE}

View file

@ -0,0 +1,42 @@
package self
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
type Self struct {
component.ComponentBase
}
func (Self) Name() string {
return "self"
}
//go:embed all:stack
//go:embed self.env
var resources embed.FS
func (self Self) Stack() component.Installable {
// TODO: Move me into config!
TARGET := "https://github.com/FAU-CDI/wisski-distillery"
if self.Config.SelfRedirect != nil { // TODO: move to config!
TARGET = self.Config.SelfRedirect.String()
}
return self.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "self.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": self.Config.DefaultHost(),
"LETSENCRYPT_HOST": self.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": self.Config.CertbotEmail,
"TARGET": TARGET,
"OVERRIDES_FILE": self.Config.SelfOverridesFile,
},
})
}

View file

@ -0,0 +1,28 @@
version: "3.7"
services:
tr:
image: ghcr.io/tkw1536/tr:latest
restart: always
volumes:
- "${OVERRIDES_FILE}:/overrides.json:ro"
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8080
VIRTUAL_PATH: /
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
# the overrides file
OVERRIDES: /overrides.json
# where to redirect to
TARGET: ${TARGET}
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,217 @@
package sql
import (
"errors"
"fmt"
"io"
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
"github.com/tkw1536/goprogram/stream"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// sqlOpen opens a new sql connection to the provided database using the administrative credentials
func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
cfg := mysql.Config{
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
DefaultStringSize: 256,
}
db, err := gorm.Open(mysql.New(cfg), config)
if err != nil {
return db, err
}
gdb, err := db.DB()
if err != nil {
return db, err
}
gdb.SetMaxIdleConns(0)
return db, nil
}
// OpenBookkeeping opens a connection to the bookkeeping database
func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
config := &gorm.Config{}
if silent {
config.Logger = logger.Default.LogMode(logger.Silent)
}
// open the database
db, err := sql.openDatabase(sql.Config.DistilleryBookkeepingDatabase, config)
if err != nil {
return nil, err
}
// load the table
table := db.Table(sql.Config.DistilleryBookkeepingTable)
if table.Error != nil {
return nil, err
}
return table, nil
}
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
// Backup makes a backup of the sql database into dest.
func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}
if code != 0 {
return errSQLBackup
}
return nil
}
// BackupAll makes a backup of all sql databases
func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}
if code != 0 {
return errSQLBackup
}
return nil
}
// OpenShell executes a mysql shell command
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
return sql.Stack().Exec(io, "sql", "mysql", argv...)
}
// WaitShell waits for the sql database to be reachable via a docker-compose shell
func (sql SQL) WaitShell() error {
n := stream.FromNil()
return wait.Wait(func() bool {
code, err := sql.OpenShell(n, "-e", "show databases;")
return err == nil && code == 0
}, sql.PollInterval, sql.PollContext)
}
// Wait waits for a connection to the bookkeeping table to suceed
func (sql SQL) Wait() error {
return wait.Wait(func() bool {
_, err := sql.OpenBookkeeping(true)
return err == nil
}, sql.PollInterval, sql.PollContext)
}
var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
func (sql SQL) Query(query string, args ...interface{}) bool {
raw := sqle.Format(query, args...)
code, err := sql.OpenShell(stream.FromNil(), "-e", raw)
return err == nil && code == 0
}
// SQLProvision provisions a new sql database and user
func (sql SQL) Provision(name, user, password string) error {
// wait for the database
if err := sql.WaitShell(); err != nil {
return err
}
// it's not a safe database name!
if !sqle.IsSafeDatabaseName(name) {
return errInvalidDatabaseName
}
// create the database and user!
if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
return errors.New("SQLProvision: Failed to create user")
}
// and done!
return nil
}
var errSQLPurgeUser = errors.New("unable to delete user")
// SQLPurgeUser deletes the specified user from the database
func (sql SQL) PurgeUser(user string) error {
if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
return errSQLPurgeUser
}
return nil
}
var errSQLPurgeDB = errors.New("unable to drop database")
// SQLPurgeDatabase deletes the specified db from the database
func (sql SQL) PurgeDatabase(db string) error {
if !sqle.IsSafeDatabaseName(db) {
return errSQLPurgeDB
}
if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") {
return errSQLPurgeDB
}
return nil
}
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe name")
var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
func (sql SQL) Bootstrap(io stream.IOStream) error {
if err := sql.WaitShell(); err != nil {
return err
}
// create the admin user
logging.LogMessage(io, "Creating administrative user")
{
username := sql.Config.MysqlAdminUser
password := sql.Config.MysqlAdminPassword
if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
return errSQLUnableToCreateUser
}
}
// create the admin user
logging.LogMessage(io, "Creating sql database")
{
if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) {
return errSQLUnsafeDatabaseName
}
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase)
if !sql.Query(createDBSQL) {
return errSQLUnableToCreate
}
}
// wait for the database to come up
logging.LogMessage(io, "Waiting for database update to be complete")
sql.Wait()
// open the database
logging.LogMessage(io, "Migrating bookkeeping table")
{
db, err := sql.OpenBookkeeping(false)
if err != nil {
return fmt.Errorf("unable to access bookkeeping table: %s", err)
}
if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
}
}
return nil
}

View file

@ -0,0 +1,38 @@
package sql
import (
"context"
"embed"
"io/fs"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
type SQL struct {
component.ComponentBase
ServerURL string // upstream server url
PollContext context.Context // context to abort polling with
PollInterval time.Duration // duration to wait for during wait
}
func (SQL) Name() string {
return "sql"
}
//go:embed all:stack
var resources embed.FS
func (ssh SQL) Stack() component.Installable {
return ssh.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
"data",
},
})
}

View file

@ -0,0 +1,35 @@
version: "3.7"
services:
sql:
image: mariadb
volumes:
- "./data/:/var/lib/mysql"
ports:
- 127.0.0.1:3306:3306
environment:
# This combination of environment variables will configure a passwordless root user
# that can only connect to the container from 'localhost'.
# This means we can only connect using 'docker-compose exec sql mysql -C '...' '.
- "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
- "MYSQL_ROOT_HOST=localhost"
restart: always
phpmyadmin:
image: phpmyadmin/phpmyadmin
environment:
- "PMA_HOST=sql"
- "HIDE_PHP_VERSION=true"
- "UPLOAD_LIMIT=100M"
# phpmyadmin running on localhost:8080 so that we can easily access the system graphically.
# By default no admin account is created, so initial shell access to make one is needed.
ports:
- 127.0.0.1:8080:80
depends_on:
- sql
restart: always
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,25 @@
package ssh
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
type SSH struct {
component.ComponentBase
}
func (SSH) Name() string {
return "ssh"
}
//go:embed all:stack
var resources embed.FS
func (ssh SSH) Stack() component.Installable {
return ssh.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
})
}

View file

@ -0,0 +1,17 @@
version: "3.7"
services:
ssh:
image: ghcr.io/tkw1536/dockersshd:latest
command: -hostkey /keys/hostkey -shell /user_shell.sh -keylabel eu.wiss-ki.barrel.authfile -userlabel eu.wiss-ki.barrel.slug -L triplestore:7200 -L phpmyadmin:80 -L sql:3306
ports:
- "2222:2222"
volumes:
- './data/keys:/keys'
- '/var/run/docker.sock:/var/run/docker.sock:ro'
restart: always
networks:
default:
name: distillery
external: true

150
internal/component/stack.go Normal file
View file

@ -0,0 +1,150 @@
// Package stack implements a docker compose stack
package component
import (
"errors"
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
"github.com/tkw1536/goprogram/stream"
)
// Stack represents a 'docker compose' stack living in a specific directory
//
// NOTE(twiesing): In the current implementation this requires a 'docker' executable on the system.
// This executable must be capable of the 'docker compose' command.
// In the future the idea is to replace this with a native docker compose client.
type Stack struct {
Dir string // Directory this Stack is located in
DockerExecutable string // Path to the native docker executable to use
}
var errStackUpdatePull = errors.New("Stack.Update: Pull returned non-zero exit code")
var errStackUpdateBuild = errors.New("Stack.Update: Build returned non-zero exit code")
// Update pulls, builds, and then optionally starts this stack.
// This does not have a direct 'docker compose' shell equivalent.
//
// See also Up.
func (ds Stack) Update(io stream.IOStream, start bool) error {
{
code, err := ds.compose(io, "pull")
if err != nil {
return err
}
if code != 0 {
return errStackUpdatePull
}
}
{
code, err := ds.compose(io, "build", "--pull")
if err != nil {
return err
}
if code != 0 {
return errStackUpdateBuild
}
}
if start {
return ds.Up(io)
}
return nil
}
var errStackUp = errors.New("Stack.Up: Up returned non-zero exit code")
// Up creates and starts the containers in this Stack.
// It is equivalent to 'docker compose up --remove-orphans --detach' on the shell.
func (ds Stack) Up(io stream.IOStream) error {
code, err := ds.compose(io, "up", "--remove-orphans", "--detach")
if err != nil {
return err
}
if code != 0 {
return errStackUp
}
return nil
}
// Exec executes an executable in the provided running service.
// It is equivalent to 'docker compose exec $service $executable $args...'.
//
// It returns the exit code of the process.
func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...string) (int, error) {
compose := []string{"exec"}
if io.StdinIsATerminal() {
compose = append(compose, "-ti")
}
compose = append(compose, service)
compose = append(compose, executable)
compose = append(compose, args...)
return ds.compose(io, compose...)
}
// Run executes the provided service with the given executable.
// It is equivalent to 'docker compose run [--rm] $service $executable $args...'.
//
// It returns the exit code of the process.
func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string, args ...string) (int, error) {
compose := []string{"run"}
if autoRemove {
compose = append(compose, "--rm")
}
if !io.StdinIsATerminal() {
compose = append(compose, "-T")
}
compose = append(compose, service, command)
compose = append(compose, args...)
code, err := ds.compose(io, compose...)
if err != nil {
return execx.ExecCommandError, nil
}
return code, nil
}
var errStackRestart = errors.New("Stack.Restart: Restart returned non-zero exit code")
// Restart restarts all containers in this Stack.
// It is equivalent to 'docker compose restart' on the shell.
func (ds Stack) Restart(io stream.IOStream) error {
code, err := ds.compose(io, "restart")
if err != nil {
return err
}
if code != 0 {
return errStackRestart
}
return nil
}
var errStackDown = errors.New("Stack.Down: Down returned non-zero exit code")
// Down stops and removes all containers in this Stack.
// It is equivalent to 'docker compose down -v' on the shell.
func (ds Stack) Down(io stream.IOStream) error {
code, err := ds.compose(io, "down", "-v")
if err != nil {
return err
}
if code != 0 {
return errStackDown
}
return nil
}
// compose executes a 'docker compose' command on this stack.
//
// NOTE(twiesing): Check if this can be replaced by an internal call to libcompose.
// But probably not.
func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
if ds.DockerExecutable == "" {
var err error
ds.DockerExecutable, err = execx.LookPathAbs("docker")
if err != nil {
return execx.ExecCommandError, err
}
}
return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
}

View file

@ -0,0 +1,57 @@
# This file is used to initialize a new GraphDB repository.
# In this file the variables ${GRAPHDB_REPO} and ${INSTANCE_DOMAIN} will be replaced.
# All other variables will be left untouched.
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
@prefix rep: <http://www.openrdf.org/config/repository#>.
@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
@prefix sail: <http://www.openrdf.org/config/sail#>.
@prefix owlim: <http://www.ontotext.com/trree/owlim#>.
[] a rep:Repository ;
rep:repositoryID "${GRAPHDB_REPO}" ;
rdfs:label "${INSTANCE_DOMAIN}" ;
rep:repositoryImpl [
rep:repositoryType "graphdb:SailRepository" ;
sr:sailImpl [
sail:sailType "graphdb:Sail" ;
owlim:owlim-license "" ;
owlim:base-URL "http://${INSTANCE_DOMAIN}/" ;
owlim:defaultNS "" ;
owlim:entity-index-size "10000000" ;
owlim:entity-id-size "32" ;
owlim:imports "" ;
owlim:repository-type "file-repository" ;
owlim:ruleset "empty" ;
owlim:storage-folder "storage" ;
owlim:enable-context-index "false" ;
owlim:cache-memory "80m" ;
owlim:tuple-index-memory "80m" ;
owlim:enablePredicateList "false" ;
owlim:predicate-memory "0%" ;
owlim:fts-memory "0%" ;
owlim:ftsIndexPolicy "never" ;
owlim:ftsLiteralsOnly "true" ;
owlim:in-memory-literal-properties "false" ;
owlim:enable-literal-index "true" ;
owlim:index-compression-ratio "-1" ;
owlim:check-for-inconsistencies "false" ;
owlim:disable-sameAs "false" ;
owlim:enable-optimization "true" ;
owlim:transaction-mode "safe" ;
owlim:transaction-isolation "true" ;
owlim:query-timeout "0" ;
owlim:query-limit-results "0" ;
owlim:throw-QueryEvaluationException-on-timeout "false" ;
owlim:useShutdownHooks "true" ;
owlim:read-only "false" ;
owlim:nonInterpretablePredicates "http://www.w3.org/2000/01/rdf-schema#label;http://www.w3.org/1999/02/22-rdf-syntax-ns#type;http://www.ontotext.com/owlim/ces#gazetteerConfig;http://www.ontotext.com/owlim/ces#metadataConfig" ;
]
].

View file

@ -0,0 +1,247 @@
package triplestore
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/fs"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
type TriplestoreUserPayload struct {
Password string `json:"password"`
AppSettings TriplestoreUserAppSettings `json:"appSettings"`
GrantedAuthorities []string `json:"grantedAuthorities"`
}
type TriplestoreUserAppSettings struct {
DefaultInference bool `json:"DEFAULT_INFERENCE"`
DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
DefaultSameas bool `json:"DEFAULT_SAMEAS"`
IgnoreSharedQueries bool `json:"IGNORE_SHARED_QUERIES"`
ExecuteCount bool `json:"EXECUTE_COUNT"`
}
// OpenRaw makes an http request to the triplestore api.
//
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
// When bodyName is empty, simply marshal body as application/json
func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
var reader io.Reader
var contentType string
// for "PUT" and "POST" we setup a body
if method == "PUT" || method == "POST" {
if bodyName != "" {
buffer := &bytes.Buffer{}
writer := multipart.NewWriter(buffer)
contentType = writer.FormDataContentType()
part, err := writer.CreateFormFile(bodyName, "filename.txt")
if err != nil {
return nil, err
}
io.Copy(part, bytes.NewReader(body.([]byte)))
writer.Close()
reader = buffer
} else {
contentType = "application/json"
mbytes, err := json.Marshal(body)
if err != nil {
return nil, err
}
reader = bytes.NewReader(mbytes)
}
}
// create the request object
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
if err != nil {
return nil, err
}
// Setup configuration!
if accept != "" {
req.Header.Set("Accept", accept)
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
// and send it
return http.DefaultClient.Do(req)
}
// Wait waits for the connection to the Triplestore to succeed.
// This is achieved using a polling strategy.
func (ts Triplestore) Wait() error {
return wait.Wait(func() bool {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
if err != nil {
return false
}
defer res.Body.Close()
return true
}, ts.PollInterval, ts.PollContext)
}
// TriplestorePurgeUser deletes the specified user from the triplestore
func (ts Triplestore) PurgeUser(user string) error {
res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
if err != nil {
return err
}
if res.StatusCode != http.StatusNoContent {
return errors.Errorf("Delete returned code %d", res.StatusCode)
}
return nil
}
// TriplestorePurgeRepo deletes the specified repo from the triplestore
func (ts Triplestore) PurgeRepo(repo string) error {
res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
if err != nil {
return err
}
if res.StatusCode != http.StatusOK {
return errors.Errorf("Delete returned code %d", res.StatusCode)
}
return nil
}
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
// TriplestoreBackup backs up the repository named repo into the writer dst.
func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
if err != nil {
return 0, err
}
if res.StatusCode != http.StatusOK {
return 0, errTSBackupWrongStatusCode
}
defer res.Body.Close()
return io.Copy(dst, res.Body)
}
type Repository struct {
ID string `json:"id"`
Title string `json:"title"`
URI string `json:"uri"`
Type string `json:"type"`
SesameType string `json:"sesameType"`
Location string `json:"location"`
Readable bool `json:"readable"`
Writable bool `json:"writable"`
Local bool `json:"local"`
}
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
if err != nil {
return nil, err
}
defer res.Body.Close()
err = json.NewDecoder(res.Body).Decode(&repos)
return
}
// TriplestoreBackup backs up every graphdb instance into dst
func (ts Triplestore) BackupAll(dst string) error {
// list all the repositories
repos, err := ts.listRepositories()
if err != nil {
return err
}
// create the base directory
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
return err
}
// iterate over all the repositories
for _, repo := range repos {
if rErr := (func(repo Repository) error {
name := filepath.Join(dst, repo.ID+".nq")
dest, err := os.Create(name)
if err != nil {
return err
}
defer dest.Close()
_, err = ts.Backup(dest, repo.ID)
return err
}(repo)); err == nil && rErr != nil {
err = rErr
}
}
return err
}
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (ts Triplestore) Bootstrap(io stream.IOStream) error {
logging.LogMessage(io, "Waiting for Triplestore")
if err := ts.Wait(); err != nil {
return err
}
logging.LogMessage(io, "Resetting admin user password")
{
res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
Password: ts.Config.TriplestoreAdminPassword,
AppSettings: TriplestoreUserAppSettings{
DefaultInference: true,
DefaultVisGraphSchema: true,
DefaultSameas: true,
IgnoreSharedQueries: false,
ExecuteCount: true,
},
GrantedAuthorities: []string{"ROLE_ADMIN"},
}, "", "")
if err != nil {
return fmt.Errorf("failed to create triplestore user: %s", err)
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
// we set the password => requests are unauthorized
// so we still need to enable security (see below!)
case http.StatusUnauthorized:
// a password is needed => security is already enabled.
// the password may or may not work, but that's a problem for later
logging.LogMessage(io, "Security is already enabled")
return nil
default:
return fmt.Errorf("failed to create triplestore user: %s", err)
}
}
logging.LogMessage(io, "Enabling Triplestore security")
{
res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
if err != nil {
return fmt.Errorf("failed to enable triplestore security: %s", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return errTriplestoreFailedSecurity
}
return nil
}
}

View file

@ -0,0 +1,75 @@
package triplestore
import (
"bytes"
"net/http"
_ "embed"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/tkw1536/goprogram/exit"
)
var errTripleStoreFailedRepository = exit.Error{
Message: "Failed to create repository: %s",
ExitCode: exit.ExitGeneric,
}
//go:embed create-repo.ttl
var createRepoTTL []byte
func (ts Triplestore) Provision(name, domain, user, password string) error {
if err := ts.Wait(); err != nil {
return err
}
// prepare the create repo request
var createRepo bytes.Buffer
err := unpack.WriteTemplate(&createRepo, map[string]string{
"GRAPHDB_REPO": name,
"INSTANCE_DOMAIN": domain,
}, bytes.NewReader(createRepoTTL))
if err != nil {
return err
}
// do the create!
{
res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo.Bytes(), "config", "")
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusCreated {
return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
}
}
// create the user and grant them access
{
res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
Password: password,
AppSettings: TriplestoreUserAppSettings{
DefaultInference: true,
DefaultVisGraphSchema: true,
DefaultSameas: true,
IgnoreSharedQueries: false,
ExecuteCount: true,
},
GrantedAuthorities: []string{
"ROLE_USER",
"READ_REPO_" + name,
"WRITE_REPO_" + name,
},
}, "", "")
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusCreated {
return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
}
}
return nil
}

View file

@ -0,0 +1,3 @@
*
!*.zip
!entrypoint.sh

View file

@ -0,0 +1,64 @@
# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container.
# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
# but has been modified for performance and security.
# This image is intended to be built like:
# docker build --build-arg graphdb_src=graphdb.zip .
# We first make a base image to base further builds on.
# We don't use alpine here, as that uses significantly slower musl instead of glibc.
FROM adoptopenjdk/openjdk11:debian-slim as base
# Create a user called graphdb
RUN useradd -ms /bin/bash graphdb
# make a base images, to add the sources to.
FROM base as sources
# install unzip
RUN apt-get update && apt-get install -y unzip
# add the source file (by default graphdb.zip) to the image
ARG src=graphdb.zip
ADD ${src} /graphdb.zip
# unpack it into a temporary directory
RUN unzip "$src" -d "/unpack/"
# Move it into /opt/graphdb, and chown it to graphdb
RUN mv "/unpack"/* /opt/graphdb
RUN chown -R graphdb:graphdb /opt/graphdb
# finally make an image that will run
FROM base as final
# add the entrypoint script
ADD entrypoint.sh /entrypoint.sh
# copy over the sources
COPY --from=sources /opt/graphdb /opt/graphdb
# set environment variables for graphdb_home and path
ENV GRAPHDB_HOME=/opt/graphdb
ENV PATH=$GRAPHDB_HOME/bin:$PATH
# Workaround for CVE-2021-44228
# (not sure if we are vulnerable, but just because)
ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
# expose a port
EXPOSE 7200
# setup a healthcheck, that checks if the server is up.
RUN apt-get update && apt-get install -y curl
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
# Add volumes for data, work and logs as these might be accessible from the outside.
# To add your own configuration, manually mount a config file into /opt/graphdb/work
VOLUME /opt/graphdb/data
VOLUME /opt/graphdb/work
VOLUME /opt/graphdb/logs
# setup command and entrypoint
CMD ["-Dgraphdb.home=/opt/graphdb"]
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

View file

@ -0,0 +1,22 @@
version: "3.7"
services:
triplestore:
build: .
ports:
- "127.0.0.1:7200:7200"
volumes:
- './data/data:/opt/graphdb/data'
- './data/work:/opt/graphdb/work'
- './data/logs:/opt/graphdb/logs'
command: "\"-Dgraphdb.home=/opt/graphdb -Ddefault.min.distinct.threshold=2G\""
# Use 1GB of heap space
environment:
GDB_HEAP_SIZE: 16G
restart: always
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,13 @@
#!/bin/bash
set -e
# Because we want to run graphdb as a limited user
# we need to make sure that the volumes are writable.
# Because of that, we 'chown'
chown graphdb:graphdb /opt/graphdb/data
chown graphdb:graphdb /opt/graphdb/work
chown graphdb:graphdb /opt/graphdb/logs
# switch to the graphdb user, and run graphdb
su graphdb -c "/opt/graphdb/bin/graphdb $@"

View file

@ -0,0 +1,43 @@
package triplestore
import (
"context"
"embed"
"io/fs"
"path/filepath"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
type Triplestore struct {
component.ComponentBase
BaseURL string // upstream server url
PollContext context.Context // context to abort polling with
PollInterval time.Duration // duration to wait for during wait
}
func (Triplestore) Name() string {
return "triplestore"
}
//go:embed all:stack
var resources embed.FS
func (ts Triplestore) Stack() component.Installable {
return ts.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{
filepath.Join("data", "data"),
filepath.Join("data", "work"),
filepath.Join("data", "logs"),
},
})
}

View file

@ -0,0 +1,52 @@
version: "3.7"
services:
nginx-proxy:
image: ghcr.io/nginx-proxy/nginx-proxy:alpine
environment:
- DEFAULT_HOST=${DEFAULT_HOST}
- HTTPS_METHOD=${HTTPS_METHOD}
ports:
- "80:80"
- "443:443"
volumes:
- "vhost:/etc/nginx/vhost.d"
- "./global.conf:/etc/nginx/conf.d/global.conf:ro"
- "./proxy.conf:/etc/nginx/proxy.conf:ro"
- "htpasswd:/etc/nginx/htpasswd"
- "html:/usr/share/nginx/html"
- "/var/run/docker.sock:/tmp/docker.sock:ro"
- "certs:/etc/nginx/certs"
labels:
com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: true
restart: always
networks:
- default
letsencrypt-nginx-proxy-companion:
image: docker.io/nginxproxy/acme-companion:latest
volumes:
- "/var/run/docker.sock:/var/run/docker.sock:ro"
- "htpasswd:/etc/nginx/htpasswd"
- "vhost:/etc/nginx/vhost.d"
- "html:/usr/share/nginx/html"
- "/var/run/docker.sock:/tmp/docker.sock:ro"
- "certs:/etc/nginx/certs"
- "acme:/etc/acme.sh"
restart: always
networks:
- default
depends_on:
- nginx-proxy
volumes:
acme:
vhost:
html:
certs:
htpasswd:
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,4 @@
# Nginx Configuration File
# These should match with distillery/resources/compose/barrel/conf/wisski.ini.
client_max_body_size 1000m;

View file

@ -0,0 +1,19 @@
# HTTP 1.1 support
proxy_http_version 1.1;
proxy_buffering off;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $proxy_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
# Mitigate httpoxy attack (see README for details)
proxy_set_header Proxy "";
# Timeouts for the proxy connection - in sync with the appropriate max_execution time.
proxy_connect_timeout 3000s;
proxy_read_timeout 3000s;
proxy_send_timeout 3000s;

View file

@ -0,0 +1,2 @@
DEFAULT_HOST=${DEFAULT_HOST}
HTTPS_METHOD=${HTTPS_METHOD}

View file

@ -0,0 +1,38 @@
package web
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
// Web implements the web component
type Web struct {
component.ComponentBase
}
func (Web) Name() string {
return "web"
}
//go:embed all:stack
//go:embed web.env
var resources embed.FS
func (web Web) Stack() component.Installable {
HTTPS_METHOD := "nohttp"
if web.Config.HTTPSEnabled() {
HTTPS_METHOD = "redirect"
}
return web.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "web.env",
EnvContext: map[string]string{
"DEFAULT_HOST": web.Config.DefaultDomain,
"HTTPS_METHOD": HTTPS_METHOD,
},
})
}

View file

@ -1,6 +1,6 @@
package config
import "github.com/FAU-CDI/wisski-distillery/internal/password"
import "github.com/FAU-CDI/wisski-distillery/pkg/password"
// NewPassword returns a new password using the password settings from this configuration
func (cfg Config) NewPassword() (string, error) {

View file

@ -6,7 +6,7 @@ import (
"strconv"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
)

View file

@ -0,0 +1,15 @@
package core
import _ "embed"
// DefaultOverridesJSON contains a template for a new 'overrides.json' file
//go:embed bootstrap/overrides.json
var DefaultOverridesJSON []byte
// DefaultAuthorizedKeys contains a template for a new 'global_authorized_keys' file
//go:embed bootstrap/global_authorized_keys
var DefaultAuthorizedKeys []byte
// ConfigFileTemplate contains a template for a new configuration file
//go:embed bootstrap/env
var ConfigFileTemplate []byte

View file

@ -0,0 +1,63 @@
# Several docker-compose files are created to manage global services and the system itself.
# On top of this all real-system space will be created under this directory.
DEPLOY_ROOT=${DEPLOY_ROOT}
# Each created Drupal Instance corresponds to a single domain name.
# These domain names should either be a complete domain name or a sub-domain of a default domain.
# This setting configures the default domain-name to create subdomains of.
DEFAULT_DOMAIN=${DEFAULT_DOMAIN}
# By default, the default domain redirects to the distillery repository.
# If you want to change this, set an alternate domain name here.
SELF_REDIRECT=
# By default, only the 'self' domain above is caught.
# To catch additional domains, add them here (comma seperated)
SELF_EXTRA_DOMAINS=
# You can override individual URLS in the homepage.
# Do this by adding URLs (without trailing '/'s) into a JSON file
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
# The system can support setting up certificate(s) automatically.
# It can be enabled by setting an email for certbot certificates.
# This email address can be configured here.
CERTBOT_EMAIL=
# The maximum age (in days) for backups to be kept.
# Backups older than this will be removed when a new backup is made.
MAX_BACKUP_AGE=30
# Each Drupal instance requires a corresponding system user, database users and databases.
# These are also set by the appropriate domain name.
# To differentiate them from other users of the system, these names can be prefixed.
# The prefix to use can be configured here.
# When changing these please consider that no system user may exist that has the same name as a mysql user.
# This is a MariaDB restriction.
MYSQL_USER_PREFIX=mysql-factory-
MYSQL_DATABASE_PREFIX=mysql-factory-
GRAPHDB_USER_PREFIX=graphdb-factory-
GRAPHDB_REPO_PREFIX=graphdb-factory-
# In addition to the filesystem the WissKI distillery requires a single SQL table.
# It uses this database to store a list of installed things
DISTILLERY_BOOKKEEPING_DATABASE=distillery
DISTILLERY_BOOKKEEPING_TABLE=distillery
# Various components use password-based-authentication.
# These passwords are generated automatically.
# This variable can be used to determine their length.
PASSWORD_LENGTH=64
# A file to be used for global authorized_keys for the ssh server.
GLOBAL_AUTHORIZED_KEYS_FILE=${AUTHORIZED_KEYS_FILE}
# The admin user and password of the GraphDB interface, to be used for queries
GRAPHDB_ADMIN_USER=${GRAPHDB_ADMIN_USER}
GRAPHDB_ADMIN_PASSWORD=${GRAPHDB_ADMIN_PASSWORD}
# The admin password to use for access to mysql
MYSQL_ADMIN_USER=${MYSQL_ADMIN_USER}
MYSQL_ADMIN_PASSWORD=${MYSQL_ADMIN_PASSWORD}

View file

@ -0,0 +1,2 @@
# This file contains authorized_keys files valid for every repository in the distillery
# The syntax of this file is easy, one key per line, empty lines or those starting with '#' are ignored

View file

@ -0,0 +1 @@
{}

14
internal/core/core.go Normal file
View file

@ -0,0 +1,14 @@
// Package core implements the core of the WissKI Distillery and the wdcli executable.
// It does not depend on any other packages.
package core
// BaseDirectoryDefault is the default deploy directory to load the distillery from.
const BaseDirectoryDefault = "/var/www/deploy"
// Executable is the name of the 'wdcli' executable.
// It should be located inside the deployment directory.
const Executable = "wdcli"
// Config file is the name of the config file.
// It should be located inside the deployment directory.
const ConfigFile = ".env"

8
internal/core/flags.go Normal file
View file

@ -0,0 +1,8 @@
package core
// Flags are global flags for the wdcli executable
type Flags struct {
ConfigPath string `short:"c" long:"config" description:"Path to distillery configuration file"`
InternalInDocker bool `long:"internal-in-docker" description:"Internal Flag to signal the shell that it is running inside a docker stack belonging to the distillery"`
}

71
internal/core/meta.go Normal file
View file

@ -0,0 +1,71 @@
package core
import (
"errors"
"io/fs"
"os"
"os/user"
"path/filepath"
"strings"
)
// MetaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable.
// It is expected to be in the current user's home directory.
//
// You probably want to use [MetaConfigPath] instead.
//
// It should contain the path to a deployment directory.
const MetaConfigFile = "." + Executable
// MetaConfigPath returns the full path to the MetaConfigPath()
func MetaConfigPath() (string, error) {
// find the current user
usr, err := user.Current()
if err != nil {
return "", err
}
return filepath.Join(usr.HomeDir, MetaConfigFile), nil
}
var errReadBaseDirectoryEmpty = errors.New("ReadBaseDirectory: Directory is empty")
// ReadBaseDirectory reads the base deployment directory from the environment.
// Use [ParamsFromEnv] to initialize parameters completely.
//
// It does not perform any reading of files.
func ReadBaseDirectory() (value string, err error) {
// get the path!
path, err := MetaConfigPath()
if err != nil {
return "", err
}
// read the meta config file!
contents, err := os.ReadFile(path)
if err != nil {
return "", err
}
// and trim the spaces!
value = strings.TrimSpace(string(contents))
// check that it is actually set!
if len(value) == 0 {
return "", errReadBaseDirectoryEmpty
}
// and return it!
return value, nil
}
// WriteBaseDirectory writes the base directory to the environment, or returns an error
func WriteBaseDirectory(dir string) error {
// get the path!
path, err := MetaConfigPath()
if err != nil {
return err
}
// just put the directory inside it!
return os.WriteFile(path, []byte(dir), fs.ModePerm)
}

31
internal/core/params.go Normal file
View file

@ -0,0 +1,31 @@
package core
import (
"os"
"path/filepath"
)
// Params are used to initialize the excutable.
type Params struct {
ConfigPath string // ConfigPath is the path to the configuration file for the distillery
}
// ParamsFromEnv creates a new set of parameters from the environment.
// Uses [ReadBaseDirectory] or falls back to [BaseDirectoryDefault].
func ParamsFromEnv() (params Params, err error) {
// try to read the base directory!
value, err := ReadBaseDirectory()
switch {
case os.IsNotExist(err):
params.ConfigPath = BaseDirectoryDefault
case err == nil:
params.ConfigPath = value
default:
return params, err
}
// and add the configuration file name to it!
params.ConfigPath = filepath.Join(params.ConfigPath, ConfigFile)
return params, nil
}

View file

@ -0,0 +1,26 @@
package core
import (
"github.com/tkw1536/goprogram"
"github.com/tkw1536/goprogram/meta"
)
// Requirements are requirements for the WissKI Distillery
type Requirements struct {
// Do we need an installed distillery?
NeedsDistillery bool
}
// AllowsFlag checks if the provided flag may be passed to fullfill this requirement
// By default it is used only for help page generation, and may be inaccurate.
func (r Requirements) AllowsFlag(flag meta.Flag) bool {
return r.NeedsDistillery
}
// Validate validates if this requirement is fullfilled for the provided global flags.
// It should return either nil, or an error of type exit.Error.
//
// Validate does not take into account AllowsOption, see ValidateAllowedOptions.
func (r Requirements) Validate(arguments goprogram.Arguments[Flags]) error {
return nil
}

9
internal/core/runtime.go Normal file
View file

@ -0,0 +1,9 @@
package core
import (
"embed"
)
// Runtime contains runtime resources to be installed into any instance
//go:embed all:runtime
var Runtime embed.FS

View file

@ -0,0 +1,2 @@
Files in this folder are utility scripts to be used from within individual WissKI instances.
They are mounted under runtime/ and should be used with care.

View file

@ -0,0 +1,16 @@
#!/bin/bash
# This utility script can be used to blindly update all dependencies to their latest versions.
# It does not perform any checking whatsoever.
# update the main modules
cd /var/www/data/project || exit 1
chmod u+rw web/sites/default/
composer update
# update the db
drush -y updatedb
# update the wisski dependencies
cd /var/www/data/project/web/modules/contrib/wisski || exit 1
composer update

View file

@ -0,0 +1,25 @@
#!/bin/sh
set -e
# read user
USER=$1
if [ -z "$USER" ]; then
echo "Usage: create_admin.sh USERNAME"
exit 1
fi
# read password
echo "Enter Password for $USER:"
read -s PASS
echo "Enter the same password again:"
read -s PASS2
if [ "$PASS" != "$PASS2" ]; then
echo "Passwords not equal"
exit 1
fi;
# create the user and add the admin role
cd /var/www/data/project/
drush user:create "$USER" --password="$PASS"
drush user-add-role administrator "$USER"

8
internal/core/runtime/cron.sh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
# This utility script can be used to run all cron tasks.
cd /var/www/data/project || exit 1
export PATH=/var/www/data/project/vendor/bin:$PATH
drush core-cron

View file

@ -0,0 +1,22 @@
#!/bin/bash
set -e
# make a temporary directory and cd into it
TEMPDIR=$(mktemp -d)
pushd "$TEMPDIR"
# curl the colorbox zip and unpack it
curl -L https://github.com/jackmoore/colorbox/archive/master.zip --output master.zip
unzip master.zip
# make the directory for libraries, and remove the old colorbox installation
chmod u+rw /var/www/data/project/web/sites/default/
mkdir -p /var/www/data/project/web/sites/default/libraries/
rm -rf /var/www/data/project/web/sites/default/libraries/colorbox
# copy over the new installation
mv colorbox-master/ /var/www/data/project/web/sites/default/libraries/colorbox
# cleanup
popd
rm -rf "$TEMPDIR"

View file

@ -0,0 +1,6 @@
#!/bin/sh
# This script can be used to repatch EasyRDF when needed.
cd /var/www/data/project/web/modules/contrib/wisski || exit 1
EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"

View file

@ -0,0 +1,6 @@
#!/bin/sh
# This script can be used to repatch EasyRDF when needed.
cd /var/www/data/project/web/modules/contrib/wisski/ || exit 1
TRIPLESTABCONTROLLER="./wisski_adapter_sparql11_pb/src/Controller/Sparql11TriplesTabController.php"
patch -N "$TRIPLESTABCONTROLLER" < "/patch/triples.patch"

View file

@ -0,0 +1,22 @@
#!/bin/bash
set -e
# read user
VERSION=$1
if [ -z "$VERSION" ]; then
echo "Usage: use_wisski.sh VERSION"
exit 1
fi
# update the main modules
cd /var/www/data/project
chmod u+rw web/sites/default/
composer require "drupal/wisski:$VERSION"
# update the wisski dependencies
pushd /var/www/data/project/web/modules/contrib/wisski
composer update
popd
# update the db
drush -y updatedb

View file

@ -0,0 +1,26 @@
#!/bin/bash
set -e
# temporarily extend permissions
chmod 777 web/sites/default
chmod 666 web/sites/default/*settings.php
chmod 666 web/sites/default/*services.yml
# update the core itself
composer require 'drupal/internal/core-recommended:^9' 'drupal/internal/core-composer-scaffold:^9' 'drupal/internal/core-project-message:^9' --update-with-dependencies --no-update
composer update
composer require 'drupal/wisski'
# update requirements for wisski!
pushd web/modules/contrib/wisski || exit 1
composer update
popd || exit 1
# run the update and clear the cache!
drush updatedb --yes
# drush cc
# and reset everything back to normal
chmod 755 web/sites/default
chmod 644 web/sites/default/*settings.php
chmod 644 web/sites/default/*services.yml

View file

@ -1,3 +0,0 @@
// Package internal contains various utility functions.
// These do not directly involve the distillery.
package internal

233
internal/env/backup.go vendored Normal file
View file

@ -0,0 +1,233 @@
package env
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
// backupDescription is a description for a backup
type BackupDescription struct {
Dest string // destination path
}
// Snapshot represents the result of generating a snapshot
type Backup struct {
Description BackupDescription
// various error states, which are ignored when creating the snapshot
ErrPanic interface{}
SQLErr error
TSErr error
ConfigFileErr error
ConfigFilesManifest map[string]error
InstanceListErr error
InstancesManifest []Snapshot
}
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
backup.Description = description
// catch anything critical that happened during the snapshot
defer func() {
backup.ErrPanic = recover()
}()
backup.run(io, dis)
return
}
var errBackupSkipFile = errors.New("<file not found>")
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
// create a wait group, and message channel
wg := &sync.WaitGroup{}
messages := make(chan string, 4)
// backup the sql
wg.Add(1)
go func() {
defer wg.Done()
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
messages <- sqlPath
sql, err := os.Create(sqlPath)
if err != nil {
backup.SQLErr = err
return
}
defer sql.Close()
// directly store the result
backup.SQLErr = dis.SQL().BackupAll(io, sql)
}()
// backup the triplestore
wg.Add(1)
go func() {
defer wg.Done()
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
messages <- tsPath
// directly store the result
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
}()
// backup configuration files
wg.Add(1)
go func() {
defer wg.Done()
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
backup.ConfigFileErr = err
return
}
files := []string{
filepath.Join(dis.Config.DeployRoot, ".env"), // TODO: put the name of the configuration file somewhere
filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
dis.Config.SelfOverridesFile,
dis.Config.GlobalAuthorizedKeysFile,
}
backup.ConfigFilesManifest = make(map[string]error, len(files))
for _, src := range files {
if !fsx.IsFile(src) {
backup.ConfigFilesManifest[src] = errBackupSkipFile
continue
}
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
// copy the config file and store the error message
messages <- src
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
}
}()
// backup instances
wg.Add(1)
go func() {
defer wg.Done()
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
backup.InstanceListErr = err
return
}
// list all instances
instances, err := dis.AllInstances()
if err != nil {
backup.InstanceListErr = err
return
}
iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
backup.InstancesManifest = make([]Snapshot, len(instances))
for i, instance := range instances {
backup.InstancesManifest[i] = func() Snapshot {
dir := filepath.Join(instancesBackupDir, instance.Slug)
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
return Snapshot{
ErrPanic: err,
}
}
messages <- dir
return instance.Snapshot(iochild, SnapshotDescription{
Dest: dir,
})
}()
}
}()
// wait for the group, then close the message channel.
go func() {
wg.Wait()
close(messages)
}()
// print out all the messages
for message := range messages {
io.Println(message)
}
}
// WriteReport writes out the report belonging to this backup.
// It is a separate function, to allow writing it indepenently of the rest.
func (backup Backup) WriteReport(io stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
io.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = fmt.Fprintf(report, "%#v\n", backup)
return err
}, io, "Writing backup report")
}
// ShouldPrune determines if a file with the provided modtime
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
}
// PruneBackups prunes all backups older than the maximum backup age
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
sPath := dis.SnapshotsArchivePath()
// list all the files
entries, err := os.ReadDir(sPath)
if err != nil {
return err
}
for _, entry := range entries {
// skip directories
if entry.IsDir() {
continue
}
// grab info about the file
info, err := entry.Info()
if err != nil {
return err
}
// check if it should be pruned!
if !dis.ShouldPrune(info.ModTime()) {
continue
}
// assemble path, and then remove the file!
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
if err := os.Remove(path); err != nil {
return err
}
}
return nil
}

126
internal/env/component.go vendored Normal file
View file

@ -0,0 +1,126 @@
package env
import (
"path/filepath"
"reflect"
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
"github.com/FAU-CDI/wisski-distillery/internal/component/resolver"
"github.com/FAU-CDI/wisski-distillery/internal/component/self"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/internal/component/web"
)
// components holds the various components of the distillery
// It is inlined into the [Distillery] struct, and initialized using [makeComponent].
type components struct {
// m protects the fields below
m sync.Mutex
// each component is only initialized once
web *web.Web
self *self.Self
resolver *resolver.Resolver
dis *dis.Dis
ssh *ssh.SSH
ts *triplestore.Triplestore
sql *sql.SQL
}
// makeComponent makes or returns a component inside the [component] struct of the distillery
//
// C is the type of component to initialize. It must be backed by a pointer, or makeComponent will panic.
//
// dis is the distillery to initialize components for
// field is a pointer to the appropriate struct field within the distillery components
// init is called with a new non-nil component to initialize it. It may be nil, to indicate no initialization is required.
//
// makeComponent returns the new or existing component instance
func makeComponent[C component.Component](dis *Distillery, field *C, init func(C)) C {
dis.components.m.Lock()
defer dis.components.m.Unlock()
// get the typeof C and make sure that it is a pointer type!
typC := reflect.TypeOf((*C)(nil)).Elem()
if typC.Kind() != reflect.Pointer {
panic("makeComponent: C must be backed by a pointer")
}
// if the component is non-nil, then it has already been initialized
if !reflect.ValueOf(*field).IsNil() {
return *field
}
// create a new element, and call the initializer (if requested)
*field = reflect.New(typC.Elem()).Interface().(C)
if init != nil {
init(*field)
}
// apply the base configuration
base := (*field).Base()
base.Config = dis.Config
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", (*field).Name())
// and eventually return it
return *field
}
// Components returns all components of the distillery
func (dis *Distillery) Components() []component.Component {
return []component.Component{
dis.Web(),
dis.Self(),
dis.Resolver(),
dis.Dis(),
dis.SSH(),
dis.Triplestore(),
dis.SQL(),
}
}
func (dis *Distillery) Web() *web.Web {
return makeComponent(dis, &dis.components.web, nil)
}
func (dis *Distillery) Self() *self.Self {
return makeComponent(dis, &dis.components.self, nil)
}
func (dis *Distillery) Resolver() *resolver.Resolver {
return makeComponent(dis, &dis.components.resolver, func(resolver *resolver.Resolver) {
resolver.ConfigName = "prefix.cfg" // TODO: Move into core?
resolver.Executable = dis.CurrentExecutable()
})
}
func (d *Distillery) Dis() *dis.Dis {
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
ddis.Executable = d.CurrentExecutable()
})
}
func (dis *Distillery) SSH() *ssh.SSH {
return makeComponent(dis, &dis.components.ssh, nil)
}
func (dis *Distillery) SQL() *sql.SQL {
return makeComponent(dis, &dis.components.sql, func(sql *sql.SQL) {
sql.ServerURL = dis.Upstream.SQL
sql.PollContext = dis.Context()
sql.PollInterval = time.Second
})
}
func (dis *Distillery) Triplestore() *triplestore.Triplestore {
return makeComponent(dis, &dis.components.ts, func(ts *triplestore.Triplestore) {
ts.BaseURL = "http://" + dis.Upstream.Triplestore
ts.PollContext = dis.Context()
ts.PollInterval = time.Second
})
}

63
internal/env/distillery.go vendored Normal file
View file

@ -0,0 +1,63 @@
package env
import (
"context"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
)
// Distillery represents an interface to the running distillery.
type Distillery struct {
// Config holds the configuration of the distillery.
// It is read directly from a configuration file.
Config *config.Config
// Upstream holds information to connect to the various running
// distillery components.
//
// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
// But for now this will just hold upstream configuration.
Upstream Upstream
// components hold references to the various components of the distillery.
components
}
// Upstream are the upstream urls connecting to the various external components.
type Upstream struct {
SQL string
Triplestore string
}
// Context returns a new Context belonging to this distillery
func (dis *Distillery) Context() context.Context {
return context.Background()
}
// ExecutablePath returns the path to the executable of this distillery.
func (dis *Distillery) ExecutablePath() string {
return filepath.Join(dis.Config.DeployRoot, core.Executable)
}
// UsingDistilleryExecutable checks if the current process
func (dis *Distillery) UsingDistilleryExecutable() bool {
exe, err := os.Executable()
if err != nil {
return false
}
return fsx.SameFile(exe, dis.ExecutablePath())
}
// CurrentExecutable returns the path to the current executable being used.
// When it does not exist, falls back to the default executable.
func (dis *Distillery) CurrentExecutable() string {
exe, err := os.Executable()
if err != nil || !fsx.IsFile(exe) {
return dis.ExecutablePath()
}
return exe
}

62
internal/env/init.go vendored Normal file
View file

@ -0,0 +1,62 @@
package env
import (
"os"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/tkw1536/goprogram/exit"
)
var errNoConfigFile = exit.Error{
ExitCode: exit.ExitGeneralArguments,
Message: "Configuration File does not exist",
}
var errOpenConfig = exit.Error{
ExitCode: exit.ExitGeneralArguments,
Message: "error loading configuration file: %s",
}
// NewDistillery creates a new distillery object from a set of parameters and requirements
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (env *Distillery, err error) {
env = &Distillery{
Upstream: Upstream{
SQL: "127.0.0.1:3306",
Triplestore: "127.0.0.1:7200",
},
}
if flags.InternalInDocker {
env.Upstream.SQL = "sql:3306"
env.Upstream.Triplestore = "triplestore:7200"
}
// if we don't need to load the config, there is nothing to do
if !req.NeedsDistillery {
return
}
// try to find the configuration file
cfg := flags.ConfigPath // command line flags first
if cfg == "" {
cfg = params.ConfigPath // then globally provided files
}
if cfg == "" {
return nil, errNoConfigFile
}
// open the config file!
f, err := os.Open(params.ConfigPath)
if err != nil {
return nil, errOpenConfig.WithMessageF(err)
}
defer f.Close()
// unmarshal the config
env.Config = &config.Config{
ConfigPath: cfg,
}
err = env.Config.Unmarshal(f)
return
}

408
internal/env/instances.go vendored Normal file
View file

@ -0,0 +1,408 @@
package env
import (
"bytes"
"embed"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/url"
"os"
"path/filepath"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/alessio/shellescape"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
var errNoBookkeeping = exit.Error{
Message: "instance %q does not exist in bookkeeping table",
ExitCode: exit.ExitGeneric,
}
var ErrInstanceNotFound = exit.Error{
Message: "instance not found",
ExitCode: exit.ExitGeneric,
}
var errSQL = exit.Error{
Message: "Unknown SQL Error %s",
ExitCode: exit.ExitGeneric,
}
// Instance returns the instance of the WissKI Distillery with the provided slug
func (dis *Distillery) Instance(slug string) (i Instance, err error) {
sql := dis.SQL()
if err := sql.Wait(); err != nil {
return i, err
}
table, err := sql.OpenBookkeeping(false)
if err != nil {
return i, err
}
// find the instance by slug
query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
switch {
case query.Error != nil:
return i, errSQL.WithMessageF(query.Error)
case query.RowsAffected == 0:
return i, ErrInstanceNotFound
default:
i.dis = dis
return i, nil
}
}
// HasInstance checks if the provided instance exists in the bookeeping table
func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
sql := dis.SQL()
if err := sql.Wait(); err != nil {
return false, err
}
table, err := sql.OpenBookkeeping(false)
if err != nil {
return false, err
}
query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok)
if query.Error != nil {
return false, errSQL.WithMessageF(query.Error)
}
return
}
// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances.
func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) {
if len(slugs) == 0 {
return dis.AllInstances()
}
return dis.InstancesWith(slugs...)
}
// AllInstances returns all instances of the WissKI Distillery in consistent order.
//
// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order.
func (dis *Distillery) AllInstances() ([]Instance, error) {
return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
return table
})
}
// InstancesWith returns all instances where the slug is in the provided list of names.
// The returned instances are reordered in a consistent order.
func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
return table.Where("slug IN ?", slugs)
})
}
// findInstances finds instance objects based on a query in the bookkeeping table
func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
sql := dis.SQL()
if err := sql.Wait(); err != nil {
return nil, err
}
// open the bookkeeping table
table, err := sql.OpenBookkeeping(false)
if err != nil {
return nil, err
}
// prepare a query
find := table
if order {
find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false})
}
if query != nil {
find = query(find)
}
// fetch bookkeeping instances
var bks []bookkeeping.Instance
find = find.Find(&bks)
if find.Error != nil {
return nil, errSQL.WithMessageF(find.Error)
}
// make proper instances
instances = make([]Instance, len(bks))
for i, bk := range bks {
instances[i].Instance = bk
instances[i].dis = dis
}
return instances, nil
}
// Instance represents a bookkeeping instance
type Instance struct {
bookkeeping.Instance
// Credentials for the drupal instance
DrupalUsername string
DrupalPassword string
dis *Distillery
}
// Update updates the bookkeeping table with this instance.
func (instance *Instance) Update() error {
db, err := instance.dis.SQL().OpenBookkeeping(false)
if err != nil {
return err
}
// it has never been created => we need to create it in the database
if instance.Instance.Created.IsZero() {
return db.Create(&instance.Instance).Error
}
// Update based on the primary key!
return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error
}
// Delete deletes this instance from the bookkeeping table
func (instance *Instance) Delete() error {
db, err := instance.dis.SQL().OpenBookkeeping(false)
if err != nil {
return err
}
// doesn't exist => nothing to delete
if instance.Instance.Created.IsZero() {
return nil
}
// delete it directly
return db.Delete(&instance.Instance).Error
}
// Shell executes a shell command inside the
func (instance Instance) Shell(io stream.IOStream, argv ...string) (int, error) {
return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...)
}
// Domain returns the full domain name of this instance
func (instance Instance) Domain() string {
return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain)
}
// URL returns the public URL of this instance
func (instance Instance) URL() *url.URL {
// setup domain and path
url := &url.URL{
Host: instance.Domain(),
Path: "/",
}
// use http or https scheme depending on if the distillery has it enabled
if instance.dis.Config.HTTPSEnabled() {
url.Scheme = "https"
} else {
url.Scheme = "http"
}
return url
}
//go:embed all:instances/barrel instances/barrel.env
var barrelResources embed.FS
// Stack represents a stack representing this instance
func (instance Instance) Stack() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: instance.FilesystemBase,
},
Resources: barrelResources,
ContextPath: filepath.Join("instances", "barrel"),
EnvPath: filepath.Join("instances", "barrel.env"),
EnvContext: map[string]string{
"DATA_PATH": filepath.Join(instance.FilesystemBase, "data"),
"SLUG": instance.Slug,
"VIRTUAL_HOST": instance.Domain(),
"LETSENCRYPT_HOST": instance.dis.Config.IfHttps(instance.Domain()),
"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
"RUNTIME_DIR": instance.dis.RuntimeDir(),
"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
},
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
MakeDirs: []string{"data", ".composer"},
TouchFiles: []string{
filepath.Join("data", "authorized_keys"),
},
}
}
//go:embed all:instances/reserve instances/reserve.env
var reserveResources embed.FS
func (instance Instance) ReserveStack() component.Installable {
return component.Installable{
Stack: component.Stack{
Dir: instance.FilesystemBase,
},
Resources: reserveResources,
ContextPath: filepath.Join("instances", "reserve"),
EnvPath: filepath.Join("instances", "reserve.env"),
EnvContext: map[string]string{
"VIRTUAL_HOST": instance.Domain(),
"LETSENCRYPT_HOST": instance.dis.Config.IfHttps(instance.Domain()),
"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
},
}
}
// Provision provisions an instance, assuming that the required databases already exist.
func (instance Instance) Provision(io stream.IOStream) error {
// create the basic st!
st := instance.Stack()
if err := st.Install(io, component.InstallationContext{}); err != nil {
return err
}
// Pull and build the stack!
if err := st.Update(io, false); err != nil {
return err
}
provisionParams := []string{
instance.Domain(),
instance.SqlDatabase,
instance.SqlUser,
instance.SqlPassword,
instance.GraphDBRepository,
instance.GraphDBUser,
instance.GraphDBPassword,
instance.DrupalUsername,
instance.DrupalPassword,
"", // TODO: DrupalVersion
"", // TODO: WissKIVersion
}
// escape the parameter
for i, param := range provisionParams {
provisionParams[i] = shellescape.Quote(param)
}
// figure out the provision script
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}
if code != 0 {
return errors.New("Unable to run provision script")
}
return nil
}
func (instance *Instance) NoPrefix() bool {
return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip"))
}
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
// PrefixConfig returns the prefix config belonging to this instance.
func (instance *Instance) PrefixConfig() (config string, err error) {
// if the user requested to skip the prefix, then don't do anything with it!
if instance.NoPrefix() {
return "", nil
}
var builder strings.Builder
// domain
builder.WriteString(instance.URL().String() + ":")
builder.WriteString("\n")
// default prefixes
wu := stream.NewIOStream(&builder, nil, nil, 0)
code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
if err != nil || code != 0 {
return "", errPrefixExecFailed
}
// custom prefixes
prefixPath := filepath.Join(instance.FilesystemBase, "prefixes")
if fsx.IsFile(prefixPath) {
prefix, err := os.Open(prefixPath)
if err != nil {
return "", err
}
defer prefix.Close()
if _, err := io.Copy(&builder, prefix); err != nil {
return "", err
}
builder.WriteString("\n")
}
// and done!
return builder.String(), nil
}
var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder")
// ExportPathbuilders writes pathbuilders into the directory dest
func (instance *Instance) ExportPathbuilders(dest string) error {
// export all the pathbuilders into the buffer
var buffer bytes.Buffer
wu := stream.NewIOStream(&buffer, nil, nil, 0)
code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php")
if err != nil || code != 0 {
return errPathbuildersExecFailed
}
// decode them as a json array
var pathbuilders map[string]string
if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil {
return err
}
// sort the names of the pathbuilders
names := maps.Keys(pathbuilders)
slices.Sort(names)
// write each into a file!
for _, name := range names {
pbxml := []byte(pathbuilders[name])
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
return err
}
}
return nil
}

10
internal/env/instances/barrel.env vendored Normal file
View file

@ -0,0 +1,10 @@
DATA_PATH=${DATA_PATH}
RUNTIME_DIR=${RUNTIME_DIR}
SLUG=${SLUG}
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}

View file

@ -0,0 +1,8 @@
# Ignore everything
*
# allow the following files:
!conf/*
!scripts/*
!patch/*
!wisskiutils/*

View file

@ -0,0 +1,28 @@
#######################
# Meta Settings
#######################
# Real path for volumes to be stored
DATA_PATH=/var/www/deploy/instances/example.slug/data
UTILS_DIR=/var/www/deploy/runtime/utils/
#######################
### Web Server settings
#######################
# the hostname for the website
VIRTUAL_HOST=example.com
# optional letsencrypt support
# when blank, ignore
LETSENCRYPT_HOST=
LETSENCRYPT_EMAIL=
### SQL settings
MYSQL_HOST=mysql
MYSQL_USER=user
MYSQL_PASS=pass
### GraphDB settings
GRAPHDB_HOST=graphdb
GRAPHDB_USER=user
GRAPHDB_PASS=pass

106
internal/env/instances/barrel/Dockerfile vendored Normal file
View file

@ -0,0 +1,106 @@
FROM docker.io/library/php:8.0-apache-bullseye
ARG COMPOSER_VERSION=2.3.8
WORKDIR /var/www
# install and enable the various required php extension
RUN apt-get update && apt-get install -y \
curl \
default-mysql-client \
git \
imagemagick \
libcurl4-openssl-dev \
libfreetype6-dev \
libicu-dev \
libjpeg62-turbo-dev \
libpng-dev \
libssh2-1-dev \
libwebp-dev \
libxml2-dev \
libxpm-dev \
sudo \
unzip \
vim \
zip \
&& \
docker-php-source extract && \
mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" && \
pear config-set php_ini "$PHP_INI_DIR/php.ini" && \
docker-php-ext-configure gd \
--enable-gd \
--with-webp \
--with-jpeg \
--with-xpm \
--with-freetype \
--enable-gd-jis-conv \
&& \
docker-php-ext-install \
curl \
gd \
intl \
mysqli \
opcache \
pdo_mysql \
soap \
xml \
&& \
pecl install xmlrpc-1.0.0RC3 && \
pecl install ssh2-1.3.1 && \
pecl install apcu-5.1.21 && \
pecl install uploadprogress-2.0.2 && \
docker-php-ext-enable \
apcu \
curl \
gd \
intl \
mysqli \
mysqli \
opcache \
pdo_mysql \
soap \
ssh2 \
uploadprogress \
xml \
xmlrpc \
&& \
docker-php-source delete
# enable the apache rewrite mod
RUN a2enmod rewrite
# install composer and add it to path
RUN curl -sS https://getcomposer.org/installer | php -- --version=$COMPOSER_VERSION && \
mv composer.phar /usr/local/bin/composer
ENV PATH "/usr/local/bin:/var/www/data/project/vendor/bin:$PATH"
# remove default configuration
RUN rm /etc/apache2/sites-available/*.conf && \
rm /etc/apache2/sites-enabled/*.conf
ADD patch/easyrdf.patch /patch/easyrdf.patch
ADD patch/triples.patch /patch/triples.patch
# Add wisski configuration
ADD conf/ports.conf /etc/apache2/ports.conf
ADD conf/wisski.conf /etc/apache2/sites-available/wisski.conf
ADD conf/wisski.ini /usr/local/etc/php/conf.d/wisski.ini
RUN a2ensite wisski
# volumes for composer
VOLUME /var/www/.composer
VOLUME /var/www/data
# Add and configure the entrypoint
ADD scripts/entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
CMD ["apache2-foreground"]
# Add the provision script and WissKI utils
ADD scripts/provision_container.sh /provision_container.sh
ADD wisskiutils/ /wisskiutils
# Add the user_shell.sh
ADD scripts/user_shell.sh /user_shell.sh
# expose port 8080
EXPOSE 8080

View file

@ -0,0 +1,4 @@
# This file configures where apache should listen.
# Because we are running as a limited user, we want to listen on a high port.
# For this we use port 8080
Listen 8080

View file

@ -0,0 +1,24 @@
<VirtualHost *:8080>
# the document root -- /var/www/data/project/web
DocumentRoot /var/www/data/project/web
<Directory /var/www/data/project/web>
# add types for .owl and .rdf
AddType application/rdf+xml .owl
AddType application/rdf+xml .rdf
# Rewrite the 'ontology' directory
RewriteEngine On
RewriteOptions InheritDownBefore
ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,END]
ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [END]
# Allow overrides of symlinks
Options Indexes FollowSymLinks
AllowOverride All
Require all granted
</Directory>
ErrorLog /dev/stderr
CustomLog /dev/stdout combined
</VirtualHost>

View file

@ -0,0 +1,14 @@
; File Uploads up to 1GB
file_uploads = On
upload_max_filesize = 1000M
post_max_size = 1000M
; Composer uses an absurd amount of memory
; 4GB ought to be enough
memory_limit = 4G
; Increase various limits for some long running WissKI operations
max_execution_time = 3000
max_input_time = 600
max_input_nesting_level = 640
max_input_vars = 10000

View file

@ -0,0 +1,33 @@
version: "3.7"
services:
barrel:
build: .
restart: always
hostname: ${VIRTUAL_HOST}.wisski
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8080
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
# label it with the current slug
labels:
eu.wiss-ki.barrel.slug: ${SLUG}
eu.wiss-ki.barrel.authfile: /var/www/.ssh/authorized_keys,/var/www/.ssh/global_authorized_keys
# volumes that are mounted
volumes:
- ${GLOBAL_AUTHORIZED_KEYS_FILE}:/var/www/.ssh/global_authorized_keys:ro
- ${DATA_PATH}/.composer:/var/www/.composer
- ${DATA_PATH}/data:/var/www/data
- ${DATA_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
- ${RUNTIME_DIR}:/runtime:ro
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,4 @@
281c281
< if (preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]+)|", $status, $m)) {
---
> if(preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]*)|", $status, $m)) {

View file

@ -0,0 +1,8 @@
100c100
< if($result->o instanceof \EasyRdf_Resource) {
---
> if($result->o instanceof \EasyRdf\Resource) {
118c118
< $object_text = $result->o->getValue();
---
> $object_text = $result->o->dumpValue('string');

View file

@ -0,0 +1,11 @@
#!/bin/bash
# This script contains
# chown the volumes to make sure they can be read and written by the limited user
chown www-data:www-data /var/www
chown www-data:www-data /var/www/.composer
chown www-data:www-data /var/www/data/
# run the original entrypoint
docker-php-entrypoint "$@"

View file

@ -0,0 +1,174 @@
#!/bin/bash
set -e
function log_info() {
echo -e "\033[1m$1\033[0m"
}
function log_ok() {
echo -e "\033[0;32m$1\033[0m"
}
log_info " => Reading configuration variables"
INSTANCE_DOMAIN="$1"
echo "INSTANCE_DOMAIN=$INSTANCE_DOMAIN"
shift 1
MYSQL_DATABASE="$1"
echo "MYSQL_DATABASE=$MYSQL_DATABASE"
MYSQL_USER="$2"
echo "MYSQL_USER=$MYSQL_USER"
MYSQL_PASSWORD="$3"
echo "MYSQL_PASSWORD=$MYSQL_PASSWORD"
shift 3
GRAPHDB_REPO="$1"
echo "GRAPHDB_REPO=$GRAPHDB_REPO"
GRAPHDB_USER="$2"
echo "GRAPHDB_USER=$GRAPHDB_USER"
GRAPHDB_PASSWORD="$3"
echo "GRAPHDB_PASSWORD=$GRAPHDB_PASSWORD"
shift 3
GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
DRUPAL_USER="$1"
echo "DRUPAL_USER=$DRUPAL_USER"
DRUPAL_PASS="$2"
echo "DRUPAL_PASS=$DRUPAL_PASS"
shift 2
DRUPAL_VERSION="$1"
echo "DRUPAL_VERSION=$DRUPAL_VERSION"
shift 1
WISSKI_VERSION="$1"
echo "WISSKI_VERSION=$WISSKI_VERSION"
shift 1
log_info " => Preparing installation environment"
BASE_DIR="/var/www/data"
COMPOSER_DIR="$BASE_DIR/project"
WEB_DIR="$COMPOSER_DIR/web"
ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
log_info " => Creating '$COMPOSER_DIR'"
mkdir -p "$COMPOSER_DIR"
cd "$COMPOSER_DIR"
# workaround for making the drupal sites directory writable
function drupal_sites_permission_workaround() {
chmod -R u+w "$WEB_DIR/sites/" || true
}
# install a module with composer and enable it with drush
# Example:
#
# composer_install_and_enable << EOF
# drupal/some_module:1.23 some_module
# drupal/other_module:2.34
# EOF
#
# Will install both modules, but only enable the first one.
function composer_install_and_enable() {
while IFS= read -r line; do
echo "$line" | (
read composer drush;
drupal_sites_permission_workaround
composer require "$composer"
if [ -n "$drush" ]; then
drush pm-enable --yes "$drush"
fi
)
done
}
# Create a new composer project.
log_info " => Creating composer project"
if [ -z "${DRUPAL_VERSION}" ]; then
composer --no-interaction create-project 'drupal/recommended-project:^9.0.0' .
else
composer --no-interaction create-project "drupal/recommended-project:$DRUPAL_VERSION" .
fi
# needed for composer > 2.2
composer --no-interaction config allow-plugins true
# Install drush so that we can automate a lot of things
log_info " => Installing 'drush'"
composer require drush/drush
# Use 'drush' to run the site-installation.
# Here we need to use the username, password and database creds we made above.
log_info " => Running drupal installation scripts"
drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
--account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
--db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@sql/${MYSQL_DATABASE}
drupal_sites_permission_workaround
# create a directory for ontologies.
log_info " => Creating '$ONTOLOGY_DIR'"
mkdir -p "$ONTOLOGY_DIR"
# Install the Wisski packages.
log_info " => Installing Wisski packages"
cd "$COMPOSER_DIR"
# install the development version when requested
if [ -z "${WISSKI_VERSION}" ]; then
composer require 'drupal/wisski'
else
composer require "drupal/wisski:$WISSKI_VERSION"
fi
# Install dependencies of WissKI
log_info " => Installing and patching Wisski dependencies"
pushd "$WEB_DIR/modules/contrib/wisski"
composer install
# Patch EasyRDF (for now)
EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
if [ -f "$EASYRDF_RESPONSE" ]; then
patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"
fi
popd
log_info " => Installing and enabling additional modules"
composer_install_and_enable << EOF
drupal/inline_entity_form:^1.0@RC
drupal/imagemagick
drupal/image_effects
drupal/colorbox
drupal/devel:^4.1 devel
drupal/geofield:^1.40 geofield
drupal/geofield_map:^2.85 geofield_map
drupal/imce:^2.4 imce
EOF
log_info " => Enable Wisski modules"
drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
drupal_sites_permission_workaround
log_info " => Setting up WissKI Salz Adapter"
drush php:script /wisskiutils/create_adapter.php "$INSTANCE_DOMAIN" "$GRAPHDB_REPO" "$GRAPHDB_HEADER"
log_info " => Updating TRUSTED_HOST_PATTERNS in settings.php"
/bin/bash /wisskiutils/set_trusted_host.sh
log_info " => Running initial cron"
drush core-cron
log_info " => Provisioning is now complete. "
log_ok "Your installation details are as follows:"
function printdetails() {
echo "URL: http://$INSTANCE_DOMAIN"
echo "Username: $DRUPAL_USER"
echo "Password: $DRUPAL_PASS"
}
printdetails
exit 0

View file

@ -0,0 +1,5 @@
#!/bin/bash
# This script is used to start a user shell inside the docker container.
cd "/var/www/data/project"
sudo -u www-data "PATH=/var/www/data/project/vendor/bin:$PATH" /bin/bash "$@"

View file

@ -0,0 +1,61 @@
<?php
/**
* This script will automatically create a WissKI Salz Adapter for use within the distillery.
* It will not update any existing adapter and is rather primitive.
*/
$argc = $_SERVER['argc']-3;
$argv = array_slice($_SERVER['argv'], 3);
// read parameters from the command line
if ($argc != 3) {
die("Usage: drush php:script create_adapter.php INSTANCE_DOMAIN GRAPHDB_REPO HEADER");
}
$INSTANCE_DOMAIN = $argv[0];
$GRAPHDB_REPO = $argv[1];
$HEADER = $argv[2];
//
// PROPERTIES FOR THE ADAPTER
//
$id = 'default'; // id
$type = 'sparql11_with_pb'; // plugin
$machine_name = 'default'; // machine-name
$label = 'Default WissKI Distillery Adapter';
$description = 'Adapter for ' . $INSTANCE_DOMAIN; // description
$writable = TRUE; // writable
$is_preferred_local_store = TRUE; // is_preferred_local_store
$header = $HEADER; // header
$read_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO; // read_url
$write_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO . '/statements'; // write_url
$is_federatable = TRUE; // is_federatable
$default_graph_uri = 'https://' . $INSTANCE_DOMAIN . '/';
$same_as_properties = ['http://www.w3.org/2002/07/owl#sameAs']; // same_as_properties
$ontology_graphs = []; // ontology_graphs
//
// Do the creation!
//
$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
$adapter = $storage->create([
"id" => $id,
"label" => $label,
"description" => $description,
]);
$adapter->setEngineConfig([
"id" => $type,
"machine-name" => $machine_name,
"header" => $header,
"writeable" => $writable,
"is_preferred_local_store" => $is_preferred_local_store,
"read_url" => $read_url,
"write_url" => $write_url,
"is_federatable" => $is_federatable,
"default_graph" => $default_graph_uri,
"same_as_properties" => $same_as_properties,
"ontology_graphs" => $ontology_graphs,
]);
$adapter->save();

View file

@ -0,0 +1,63 @@
<?php
/**
* This script will list all the URIs that this system is aware of.
* This works by listing all the default graph uris of all the adapters.
*/
use Drupal\wisski_pathbuilder\Entity\WisskiPathEntity;
// load all the pathbuilders
$pbs = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->loadMultiple();
// map over the pathbuilders
$xmls = array_map(function($pb) {
$xml = new \SimpleXMLElement("<pathbuilderinterface></pathbuilderinterface>");
$paths = $pb->getAllPaths();
foreach ($paths as $key => $path) {
$id = $path->getID();
$path = $pb->getPbPath($id);
$pathChild = $xml->addChild("path");
$pathObject = WisskiPathEntity::load($id);
foreach ($path as $subkey => $value) {
if (in_array($subkey, ['relativepath'])) {
continue;
}
if ($subkey == "parent") {
$subkey = "group_id";
}
$pathChild->addChild($subkey, htmlspecialchars($value));
}
$pathArray = $pathChild->addChild('path_array');
foreach ($pathObject->getPathArray() as $subkey => $value) {
$pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value);
}
$pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty()));
$pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName()));
$pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb()));
$pathChild->addChild('description', htmlspecialchars($pathObject->getDescription()));
$pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid()));
if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") {
$pathChild->addChild('is_group', "1");
} else {
$pathChild->addChild('is_group', "0");
}
$pathChild->addChild('name', htmlspecialchars($pathObject->getName()));
}
// turn it into XML
$dom = dom_import_simplexml($xml)->ownerDocument;
$dom->formatOutput = TRUE;
return $dom->saveXML();
}, $pbs);
echo json_encode($xmls);

View file

@ -0,0 +1,19 @@
<?php
/**
* This script will list all the URIs that this system is aware of.
* This works by listing all the default graph uris of all the adapters.
*/
// iterate over all adapters
$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
foreach ($storage->loadMultiple() as $adapter) {
// read the configuration, and check if we have a default graph
$conf = $adapter->getEngine()->getConfiguration();
if(!array_key_exists('default_graph', $conf)) {
continue;
}
// and echo it out
echo $conf['default_graph'] . "\n";
}

View file

@ -0,0 +1,13 @@
#!/bin/bash
# This utility script can be used to configure the trusted host settings inside of settings.php.
# It doesn't take care of corner cases and should only be used when needed.
INSTANCE_DOMAIN="$(hostname -f)"
INSTANCE_DOMAIN="${INSTANCE_DOMAIN%.wisski}"
TRUSTED_HOST_PATTERN="${INSTANCE_DOMAIN//\./\\\\.}"
TRUSTED_HOST_PATTERNS='["'$TRUSTED_HOST_PATTERN'"]'
echo "Setting 'trusted_host_patterns' to $TRUSTED_HOST_PATTERNS"
bash /wisskiutils/settings_php_set.sh 'trusted_host_patterns' "$TRUSTED_HOST_PATTERNS"

View file

@ -0,0 +1,17 @@
#!/bin/bash
# settings_php_get.sh name
# Gets the 'settings_php_get.php' setting 'name' as json-encoded value, or null when it does not exist.
NAME=$1
if [ -z "$NAME" ]; then
echo "Usage: get_settings_setting.sh NAME"
exit 1
fi;
echo "$NAME" | drush php:eval '
use \Drupal\Core\Site\Settings;
$name=trim(file_get_contents("php://stdin"));
echo json_encode(Settings::get($name));
';

View file

@ -0,0 +1,56 @@
#!/bin/bash
# settings_php_set.sh name value
# Sets the 'settings.php' setting 'name' to 'value'.
# Value must be json-encoded.
NAME=$1
VALUE=$2
if [ -z "$NAME" ]; then
echo "Usage: settings_php_set.sh NAME VALUE"
exit 1
fi;
if [ -z "$VALUE" ]; then
echo "Usage: settings_php_set.sh NAME VALUE"
exit 1
fi;
cd /var/www/data/project
chmod u+w web/sites/default/settings.php
(echo "$NAME"; echo "$VALUE" ) | drush php:eval '
include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
// read NAME and VALUE from STDIN
$content=file_get_contents("php://stdin");
$newline=strpos($content, "\n");
$name=trim(substr($content, 0, $newline));
$jvalue=trim(substr($content, $newline + 1));
// decode json values
$value = @json_decode($jvalue);
if ($value === null && json_last_error() !== JSON_ERROR_NONE) {
echo "Invalid JSON, cannot update settings.php. \n";
return 1;
}
// make parameters to drush_rewrite_settings
$settings["settings"][$name] = (object)[
"value" => $value,
"required" => TRUE,
];
// find the actual settings.php file to rewrite
$filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
drupal_rewrite_settings($settings, $filename);
echo "Wrote " . $filename . "\n";
return 0;
';
EXIT=$?
chmod u-w web/sites/default/settings.php
exit $?

4
internal/env/instances/reserve.env vendored Normal file
View file

@ -0,0 +1,4 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}

View file

@ -0,0 +1,26 @@
version: "3.7"
services:
static:
image: tkw01536/gostatic
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8043
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
ports:
- 8043
# volumes that are mounted
volumes:
- ./index.html:/srv/http/index.html:ro
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,4 @@
<!DOCTYPE html>
This domain name is reserved.
Content is a work in progress.

90
internal/env/instances_provision.go vendored Normal file
View file

@ -0,0 +1,90 @@
package env
import (
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
"github.com/pkg/errors"
)
func (dis *Distillery) InstancesDir() string {
return filepath.Join(dis.Config.DeployRoot, "instances")
}
func (dis *Distillery) InstanceDir(slug string) string {
return filepath.Join(dis.InstancesDir(), slug)
}
func (dis *Distillery) InstanceSQL(slug string) (database, user string) {
database = dis.Config.MysqlDatabasePrefix + slug
user = dis.Config.MysqlUserPrefix + slug
return
}
func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) {
repo = dis.Config.GraphDBRepoPrefix + slug
user = dis.Config.GraphDBUserPrefix + slug
return
}
var errInvalidSlug = errors.New("Not a valid slug")
// NewInstance fills the struct for a new distillery instance.
// It validates that slug is a valid name for an instance.
//
// It does not perform any checks if the instance already exists, or does the creation in the database.
func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
// make sure that the slug is valid!
if _, err := config.IsValidSlug(slug); err != nil {
return i, errInvalidSlug
}
// generate sql data
sqlPassword, err := dis.Config.NewPassword()
if err != nil {
return i, err
}
sqlDB, sqlUser := dis.InstanceSQL(slug)
// generate ts data
tsPassword, err := dis.Config.NewPassword()
if err != nil {
return i, err
}
tsRepo, tsUser := dis.InstanceGraphDB(slug)
// generate drupal data
drPassword, err := dis.Config.NewPassword()
if err != nil {
return i, err
}
drUser := "admin"
// make the instance object!
instance := bookkeeping.Instance{
Slug: slug,
OwnerEmail: "",
AutoBlindUpdateEnabled: true,
FilesystemBase: dis.InstanceDir(slug),
SqlDatabase: sqlDB,
SqlUser: sqlUser,
SqlPassword: sqlPassword,
GraphDBRepository: tsRepo,
GraphDBUser: tsUser,
GraphDBPassword: tsPassword,
}
i.DrupalUsername = drUser
i.DrupalPassword = drPassword
// store the instance in the object and return it!
i.Instance = instance
i.dis = dis
return i, nil
}

8
internal/env/runtime.go vendored Normal file
View file

@ -0,0 +1,8 @@
package env
import "path/filepath"
// RuntimeDir returns the path to the runtime directory
func (dis *Distillery) RuntimeDir() string {
return filepath.Join(dis.Config.DeployRoot, "runtime")
}

33
internal/env/server.go vendored Normal file
View file

@ -0,0 +1,33 @@
package env
import (
"io"
"net/http"
)
// TODO: Move this into dis!
// Server represents a server for this distillery
type Server struct {
dis *Distillery
}
func (dis *Distillery) Server() *Server {
return &Server{
dis: dis,
}
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
instances, err := s.dis.AllInstances()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
io.WriteString(w, "Something went wrong")
return
}
w.WriteHeader(http.StatusOK)
for _, instance := range instances {
io.WriteString(w, instance.Slug+"\n")
}
}

253
internal/env/snapshot.go vendored Normal file
View file

@ -0,0 +1,253 @@
package env
import (
"fmt"
"io/fs"
"os"
"path/filepath"
"sync"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/password"
"github.com/tkw1536/goprogram/stream"
)
// SnapshotsDir returns the path that contains all snapshot related data.
func (dis *Distillery) SnapshotsDir() string {
return filepath.Join(dis.Config.DeployRoot, "snapshots")
}
// SnapshotsStagingPath returns the path to the directory containing a temporary staging area for snapshots.
// Use NewSnapshotStagingDir to generate a new staging area.
func (dis *Distillery) SnapshotsStagingPath() string {
return filepath.Join(dis.SnapshotsDir(), "staging")
}
// SnapshotsArchivePath returns the path to the directory containing all exported archives.
// Use NewSnapshotArchivePath to generate a path to a new archive in this directory.
func (dis *Distillery) SnapshotsArchivePath() string {
return filepath.Join(dis.SnapshotsDir(), "archives")
}
// NewSnapshotArchivePath returns the path to a new archive with the provided prefix.
// The path is guaranteed to not exist.
func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
// TODO: Consider moving these into a subdirectory with the provided prefix.
for path == "" || fsx.Exists(path) {
name := dis.newSnapshotName(prefix) + ".tar.gz"
path = filepath.Join(dis.SnapshotsArchivePath(), name)
}
return
}
// newSnapshot name returns a new basename for a snapshot with the provided prefix.
// The name is guaranteed to be unique within this process.
func (*Distillery) newSnapshotName(prefix string) string {
suffix, _ := password.Password(64) // silently ignore any errors!
if prefix == "" {
prefix = "backup"
} else {
prefix = "snapshot-" + prefix
}
return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
}
// NewSnapshotStagingDir returns the path to a new snapshot directory.
// The directory is guaranteed to have been freshly created.
func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
for path == "" || os.IsExist(err) {
path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
err = os.Mkdir(path, os.ModeDir)
}
if err != nil {
path = ""
}
return
}
// SnapshotDescription is a description for a snapshot
type SnapshotDescription struct {
Dest string // destination path
Keepalive bool // should we keep the instance alive while making the snapshot?
}
// Snapshot represents the result of generating a snapshot
type Snapshot struct {
Description SnapshotDescription
Instance bookkeeping.Instance
// various error states, which are ignored when creating the snapshot
ErrPanic interface{} // panic, if any
ErrStart error
ErrStop error
ErrBookkeep error
ErrPathbuilder error
ErrFilesystem error
ErrTriplestore error
ErrSSQL error
}
// Snapshot creates a new snapshot of this instance into dest
func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
// setup the snapshot
snapshot.Description = desc
snapshot.Instance = instance.Instance
// catch anything critical that happened during the snapshot
defer func() {
snapshot.ErrPanic = recover()
}()
// and do the create!
snapshot.create(io, instance)
return
}
func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
stack := instance.Stack()
// stop the instance (unless it was explicitly asked to not do so!)
if !snapshot.Description.Keepalive {
logging.LogMessage(io, "Stopping instance")
snapshot.ErrStop = stack.Down(io)
defer func() {
logging.LogMessage(io, "Starting instance")
snapshot.ErrStart = stack.Up(io)
}()
}
// create a wait group, and message channel
wg := &sync.WaitGroup{}
messages := make(chan string, 4)
// write bookkeeping information
wg.Add(1)
go func() {
defer wg.Done()
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
messages <- bkPath
info, err := os.Create(bkPath)
if err != nil {
snapshot.ErrBookkeep = err
return
}
defer info.Close()
// print whatever is in the database
// TODO: This should be sql code, maybe gorm can do that?
_, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance)
}()
// write pathbuilders
wg.Add(1)
go func() {
defer wg.Done()
pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
messages <- pbPath
// create the directory!
if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
snapshot.ErrPathbuilder = err
return
}
// put in all the pathbuilders
snapshot.ErrPathbuilder = instance.ExportPathbuilders(pbPath)
}()
// backup the filesystem
wg.Add(1)
go func() {
defer wg.Done()
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
snapshot.ErrFilesystem = err
return
}
// copy over whatever is in the base directory
snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
messages <- dst
})
}()
// backup the graph db repository
wg.Add(1)
go func() {
defer wg.Done()
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
messages <- tsPath
nquads, err := os.Create(tsPath)
if err != nil {
snapshot.ErrTriplestore = err
}
defer nquads.Close()
// directly store the result
_, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
}()
// backup the sql database
wg.Add(1)
go func() {
defer wg.Done()
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
messages <- sqlPath
sql, err := os.Create(sqlPath)
if err != nil {
snapshot.ErrSSQL = err
return
}
defer sql.Close()
// directly store the result
snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
}()
// TODO: Backup the docker image
// wait for the group, then close the message channel.
go func() {
wg.Wait()
close(messages)
}()
// print out all the messages
for message := range messages {
io.Println(message)
}
}
// WriteReport writes out the report belonging to this snapshot.
// It is a separate function, to allow writing it indepenently of the rest.
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
return logging.LogOperation(func() error {
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
io.Println(reportPath)
// create the report file!
report, err := os.Create(reportPath)
if err != nil {
return err
}
defer report.Close()
// print the report into it!
_, err = fmt.Fprintf(report, "%#v\n", snapshot)
return err
}, io, "Writing snapshot report")
}

View file

@ -1,46 +0,0 @@
// Package execx defines extensions to the "os/exec" package
package execx
import (
"os/exec"
"github.com/tkw1536/goprogram/stream"
)
// ExecCommandError is returned by Exec when a command could not be executed.
// This typically hints that the executable cannot be found, but may have other causes.
const ExecCommandError = 127
// Exec executes a system command with the specified input/output streams, working directory, and arguments.
//
// If the command executes, it's exit code will be returned.
// If the command can not be executed, returns [ExecCommandError].
func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
// setup the command
cmd := exec.Command(exe, argv...)
cmd.Dir = workdir
cmd.Stdin = io.Stdin
cmd.Stdout = io.Stdout
cmd.Stderr = io.Stderr
// run it
err := cmd.Run()
// non-zero exit
if err, ok := err.(*exec.ExitError); ok {
return err.ExitCode()
}
// unknown error
if err != nil {
return ExecCommandError
}
// everything is fine!
return 0
}
// MustExec is like Exec, except that it returns true if the command exited successfully, and else false.
func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool {
return Exec(io, workdir, exe, argv...) == 0
}

View file

@ -1,15 +0,0 @@
package execx
import (
"os/exec"
"path/filepath"
)
// LookPathAbs is like [exec.LookPath], but always returns an absolute path
func LookPathAbs(file string) (string, error) {
path, err := exec.LookPath(file)
if err != nil {
return "", err
}
return filepath.Abs(path)
}

View file

@ -1,107 +0,0 @@
package fsx
import (
"errors"
"io"
"os"
"path/filepath"
)
var ErrCopySameFile = errors.New("src and dst must be different files")
// CopyFile copies a file from src to dst.
// When dst and src are the same file, returns ErrCopySameFile.
func CopyFile(dst, src string) error {
if SameFile(src, dst) {
return ErrCopySameFile
}
// open the source
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
// stat it to get the mode!
srcStat, err := srcFile.Stat()
if err != nil {
return err
}
// open or create the destination
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode())
if err != nil {
return err
}
defer dstFile.Close()
// and do the copy!
_, err = io.Copy(dstFile, srcFile)
return err
}
var ErrCopyNoDirectory = errors.New("dst is not a directory")
// CopyDirectory copies the directory src to dst recursively.
// The destination directory must exist, or an error is returned.
//
// onCopy, when not nil, is called for each file or directory being copied.
func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
// sanity checks
if SameFile(src, dst) {
return ErrCopySameFile
}
if !IsDirectory(dst) {
return ErrCopyNoDirectory
}
// call onCopy for this directory!
if onCopy != nil {
onCopy(dst, src)
}
// iterate over the entries or bail out
entries, err := os.ReadDir(src)
if err != nil {
return err
}
for _, entry := range entries {
name := entry.Name()
eDest := filepath.Join(dst, name)
eSrc := filepath.Join(src, name)
// it is not a directory => Use CopyFile
if !entry.IsDir() {
if onCopy != nil {
onCopy(eDest, eSrc)
}
// do the copy!
if err := CopyFile(eDest, eSrc); err != nil {
return err
}
continue
}
// find out the mode of the entry
eInfo, err := entry.Info()
if err != nil {
return err
}
// make the target directory
if err := os.Mkdir(eDest, eInfo.Mode()); err != nil {
return err
}
// do the copy!
if err := CopyDirectory(eDest, eSrc, onCopy); err != nil {
return err
}
}
return nil
}

View file

@ -1,87 +0,0 @@
package fsx
import (
"os"
"path/filepath"
)
// SameFile checks if path1 and path2 refer to the same file.
// If both files exist, they are compared using [os.SameFile].
// If both files do not exist, the paths are first compared syntactically and then via recursion on [filepath.Dir].
func SameFile(path1, path2 string) bool {
// initial attempt: check if directly
same, certain := couldBeSameFile(path1, path2)
if certain {
return same
}
// second attempt: find the directory names and base paths
d1, n1 := filepath.Dir(path1), filepath.Base(path1)
d2, n2 := filepath.Dir(path2), filepath.Base(path2)
// if we have different file names (and they don't exist)
// we don't need to continue
if n1 != n2 {
return false
}
// compare the base names!
{
same, _ := couldBeSameFile(d1, d2)
return same
}
}
// couldBeSameFile checks if path1 might be the same as path2.
//
// If both files exist, compares using [os.SameFile].
// Otherwise compares absolute paths using string comparison.
//
// same indicates if they might be the same file.
// authorative indiciates if the result is authorative.
func couldBeSameFile(path1, path2 string) (same, authorative bool) {
{
// stat both files
info1, err1 := os.Stat(path1)
info2, err2 := os.Stat(path2)
// both files exist => check using os.SameFile
// the result is always authorative
if err1 == nil && err2 == nil {
same = os.SameFile(info1, info2)
authorative = true
return
}
// only 1 file errored => they could be different
if (err1 == nil) != (err2 == nil) {
return
}
// only 1 file does not exist => they could be different
if os.IsNotExist(err1) != os.IsNotExist(err2) {
return
}
}
{
// resolve paths absolutely
rpath1, err1 := filepath.Abs(path1)
rpath2, err2 := filepath.Abs(path2)
// if either path could not be resolved absolutely
// fallback to just using clean!
if err1 != nil {
rpath1 = filepath.Clean(path1)
}
if err2 != nil {
rpath2 = filepath.Clean(path2)
}
// compare using strings
same = rpath1 == rpath2
authorative = same // positive result is authorative!
return
}
}

View file

@ -1,29 +0,0 @@
package fsx
import (
"os"
"time"
)
// Touch touches a file.
// It is similar to the unix 'touch' command.
//
// If the file does not exist exists, it is created using [os.Create].
// If the file does exist, it's access and modification times are updated to the current time.
func Touch(path string) error {
_, err := os.Stat(path)
switch {
case os.IsNotExist(err):
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
return nil
case err != nil:
return err
default:
now := time.Now().Local()
return os.Chtimes(path, now, now)
}
}

View file

@ -1,24 +0,0 @@
// Package fsx provides convenient abstractions to work with the filesystem.
package fsx
import (
"os"
)
// Exists checks if the given path exists
func Exists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// IsDirectory checks if the provided path exists and is a directory
func IsDirectory(path string) bool {
info, err := os.Stat(path)
return err == nil && info.Mode().IsDir()
}
// IsFile checks if the provided path exists and is a regular file
func IsFile(path string) bool {
info, err := os.Stat(path)
return err == nil && info.Mode().IsRegular()
}

View file

@ -1,32 +0,0 @@
// Package hostname provides the hostname.
package hostname
import (
"os"
"github.com/Showmax/go-fqdn"
)
// FQDN attempts to return the fully qualified domain name of the host system.
// If an error occurs, may fall back to the empty string.
func FQDN() string {
// try the hostname function
{
fqdn, err := fqdn.FqdnHostname()
if err == nil {
return fqdn
}
}
// fallback to os hostname
{
hostname, err := os.Hostname()
if err == nil {
return hostname
}
}
// use the empty string
return ""
}

View file

@ -1,59 +0,0 @@
package logging
import (
"sync"
"github.com/tkw1536/goprogram/stream"
)
var logLevelMutex sync.Mutex
var logLevelMap = make(map[uintptr]int)
func getIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
id, ok := logID(io)
if !ok {
return 0
}
return logLevelMap[id]
}
func incIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
id, ok := logID(io)
if !ok { // if we don't have an id, then inc statically returns 1
return 1
}
logLevelMap[id]++
return logLevelMap[id]
}
func decIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
id, ok := logID(io)
if !ok { // if we don't have an id, then dec statically returns 0
return 0
}
logLevelMap[id]--
if logLevelMap[id] < 0 {
panic("DecLogIdent: decrease below 0")
}
return logLevelMap[id]
}
func logID(io stream.IOStream) (uintptr, bool) {
file, ok := io.Stdin.(interface{ Fd() uintptr })
if !ok {
return 0, false
}
return file.Fd(), true
}

View file

@ -1,30 +0,0 @@
package logging
import (
"strings"
"github.com/tkw1536/goprogram/stream"
)
// LogOperation logs a message that is displayed to the user, and then increases the log indent level.
func LogOperation(operation func() error, io stream.IOStream, format string, args ...interface{}) error {
logOperation(io, getIndent(io), format, args...)
incIndent(io)
defer decIndent(io)
return operation()
}
// LogMessage logs a message that is displayed to the user
func LogMessage(io stream.IOStream, format string, args ...interface{}) (int, error) {
return logOperation(io, getIndent(io), format, args...)
}
func logOperation(io stream.IOStream, indent int, format string, args ...interface{}) (int, error) {
message := "\033[1m" + strings.Repeat(" ", indent+1) + "=> " + format + "\033[0m\n"
if !io.StdinIsATerminal() {
message = " => " + format
}
return io.Printf(message, args...)
}

View file

@ -1,46 +0,0 @@
// Package password allows generating random passwords
package password
import (
"crypto/rand"
"math/big"
"strings"
)
// NOTE(twiesing): A bunch of scripts cannot properly handle the extra characters in the password.
// For now it is disabled, but it should be re-enabled later.
const PasswordCharSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // + "!@#$%&*"
var passwordCharCount = big.NewInt(int64(len(PasswordCharSet)))
// Password returns a randomly generated string with the provided length.
// It consists of alphanumeric characters only.
//
// When an error occurs, it is guaranteed to return "", err.
// [rand.Reader] is used as the source of randomness.
func Password(length int) (string, error) {
if length < 0 {
panic("length < 0")
}
// create a buffer to write the string to!
var password strings.Builder
password.Grow(length)
for i := 0; i < length; i++ {
// grab a random bIndex!
bIndex, err := rand.Int(rand.Reader, passwordCharCount)
if err != nil {
return "", err
}
// and use that index!
index := int(bIndex.Int64())
if err := password.WriteByte(PasswordCharSet[index]); err != nil {
return "", err
}
}
// return the password!
return password.String(), nil
}

File diff suppressed because one or more lines are too long

View file

@ -1,10 +0,0 @@
package sqle
import (
"github.com/feiin/sqlstring"
)
// Format formats the provided query with the given parameters.
func Format(query string, params ...interface{}) string {
return sqlstring.Format(query, params...)
}

View file

@ -1,80 +0,0 @@
// Package targz provides facilities for packaging tar.gz files
package targz
import (
"archive/tar"
"compress/gzip"
"io"
"io/fs"
"os"
"path/filepath"
)
// Package packages the source directory into a 'tar.gz' file into destination.
// If the destination already exists, it is truncated.
//
// onCopy, when not nil, is called for each file being copied into the archive.
func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
// create the target archive
archive, err := os.Create(dst)
if err != nil {
return 0, err
}
defer archive.Close()
// create a gzip writer
zipHandle := gzip.NewWriter(archive)
defer zipHandle.Close()
// create a tar writer
tarHandle := tar.NewWriter(zipHandle)
defer tarHandle.Close()
// and walk through it!
err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// determine the relative path
var relpath string
relpath, err = filepath.Rel(src, path)
if err != nil {
return err
}
if onCopy != nil {
onCopy(relpath, path)
}
// create a file info header!
tInfo, err := tar.FileInfoHeader(info, relpath)
if err != nil {
return err
}
tInfo.Name = filepath.ToSlash(relpath)
// write it!
if err := tarHandle.WriteHeader(tInfo); err != nil {
return err
}
// a directory => no more writing required
if info.IsDir() {
return nil
}
// open the file
handle, err := os.Open(path)
if err != nil {
return err
}
defer handle.Close()
// and copy it into the archive
ccount, err := io.Copy(tarHandle, handle)
count += ccount
return err
})
return
}

View file

@ -1,131 +0,0 @@
package unpack
import (
"io"
"io/fs"
"os"
"path/filepath"
"github.com/pkg/errors"
)
var errExpectedFileButGotDirectory = errors.New("expected a file, but got a directory")
var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a file")
// InstallDir installs the directory at src within fsys to dst.
//
// onInstallFile is called for each file or directory being installed.
//
// If the destination path does not exist, it is created using [os.MakeDirs]
// The directory is installed recursively.
func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the source file
srcFile, err := fsys.Open(src)
if err != nil {
return err
}
// stat it!
srcInfo, err := srcFile.Stat()
if err != nil {
return err
}
// make sure it's a file!
if !srcInfo.IsDir() {
return errExpectedDirectoryButGotFile
}
// call the hook (if any)
if onInstallFile != nil {
onInstallFile(dst, src)
}
// do the installation of the directory.
// the type cast should be safe.
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
// installResource installs the resource at src within fsys to dst.
//
// OnInstallFile is called for each source and destination file.
// OnInstallFile may be nil.
func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the srcFile
srcFile, err := fsys.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
// stat it!
srcInfo, err := srcFile.Stat()
if err != nil {
return err
}
// call the hook (if any)
if onInstallFile != nil {
onInstallFile(dst, src)
}
// this is a directory, so the cast is safe!
if srcInfo.IsDir() {
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
// this is a regular file!
return installFile(dst, srcInfo, srcFile)
}
func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case !dstStat.IsDir():
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
}
// NOTE(twiesing): We don't use fs.Walk here.
// If we did, we'd have to reconstruct relative paths.
// That would be very ugly!
// read the directory
entries, err := srcFile.ReadDir(-1)
if err != nil {
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
}
// iterate over all the children
for _, entry := range entries {
if err := installResource(
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
fsys,
onInstallFile,
); err != nil {
return err
}
}
return nil
}
func installFile(dst string, srcInfo fs.FileInfo, src fs.File) error {
// create the file using the right mode!
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
if err != nil {
return err
}
defer file.Close()
// copy over the content!
_, err = io.Copy(file, src)
return errors.Wrapf(err, "Error writing to destination %s", dst)
}

Some files were not shown because too many files have changed in this diff Show more