Refactor Package structure
This commit cleans up the package structure, to make two new top-level packages `internal` (for internal-use packages) and `pkg` (for general shared utility code).
This commit is contained in:
parent
487ce09979
commit
a360324f62
124 changed files with 97 additions and 101 deletions
|
|
@ -1,68 +0,0 @@
|
|||
// Package bookkeeping implements reading and writing from the bookkeeping table
|
||||
package bookkeeping
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Instance is a WissKI Instance inside the bookkeeping table.
|
||||
// It does not represent a running instance; it does not perform any validation.
|
||||
type Instance struct {
|
||||
// NOTE: Modifying this struct requires a database migration.
|
||||
// This should nnever be done unless you know what you're doing.
|
||||
|
||||
// Primary key for the instance
|
||||
Pk uint `gorm:"column:pk;primaryKey"`
|
||||
|
||||
// time the instance was created
|
||||
Created time.Time `gorm:"column:created;autoCreateTime"`
|
||||
|
||||
// slug of the system
|
||||
Slug string `gorm:"column:slug;not null;unique"`
|
||||
|
||||
// email address of the system owner (if any)
|
||||
OwnerEmail string `gorm:"column:owner_email;type:varchar(320)"`
|
||||
|
||||
// should we automatically enable updates for the system?
|
||||
AutoBlindUpdateEnabled SQLBit1 `gorm:"column:auto_blind_update_enabled;default:1"`
|
||||
|
||||
// The filesystem path the system can be found under
|
||||
FilesystemBase string `gorm:"column:filesystem_base;not null"`
|
||||
|
||||
// SQL Database credentials for the system
|
||||
SqlDatabase string `gorm:"column:sql_database;not null"`
|
||||
SqlUser string `gorm:"column:sql_user;not null"`
|
||||
SqlPassword string `gorm:"column:sql_password;not null"`
|
||||
|
||||
// GraphDB Repository
|
||||
GraphDBRepository string `gorm:"column:graphdb_repository;not null"`
|
||||
GraphDBUser string `gorm:"column:graphdb_user;not null"`
|
||||
GraphDBPassword string `gorm:"column:graphdb_password;not null"`
|
||||
}
|
||||
|
||||
func (i Instance) IsBlindUpdateEnabled() bool {
|
||||
return bool(i.AutoBlindUpdateEnabled)
|
||||
}
|
||||
|
||||
// SQLBit1 implements a boolean as a BIT(1)
|
||||
type SQLBit1 bool
|
||||
|
||||
func (sb SQLBit1) Value() (driver.Value, error) {
|
||||
if sb {
|
||||
return []byte{1}, nil
|
||||
} else {
|
||||
return []byte{0}, nil
|
||||
}
|
||||
}
|
||||
|
||||
var errBadBool = errors.New("SQLBit1: Database does not contain Bit(1)")
|
||||
|
||||
func (sb *SQLBit1) Scan(src interface{}) error {
|
||||
if bytes, ok := src.([]byte); ok && len(bytes) == 1 {
|
||||
*sb = bytes[0] == 1
|
||||
return nil
|
||||
}
|
||||
return errBadBool
|
||||
}
|
||||
69
internal/component/component.go
Normal file
69
internal/component/component.go
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
// Package component holds the main abstraction for components.
|
||||
package component
|
||||
|
||||
import (
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||
)
|
||||
|
||||
// Component represents a logical subsystem of the distillery.
|
||||
//
|
||||
// By convention these are defined within their corresponding subpackage.
|
||||
// This subpackage also contains all required resources.
|
||||
// Furthermore, a component is typically instantiated using a call on the ["distillery.Distillery"] struct.
|
||||
//
|
||||
// Each Component should make use of [ComponentBase] for sane defaults.
|
||||
//
|
||||
// For example, the web.Web component lives in the web package and can be created like:
|
||||
//
|
||||
// var dis Distillery
|
||||
// web := dis.Web()
|
||||
type Component interface {
|
||||
// Name returns the name of this component.
|
||||
// It should correspond to the appropriate subpackage.
|
||||
Name() string
|
||||
|
||||
// Path returns the path this component is installed at.
|
||||
// By convention it is /var/www/deploy/internal/core/${Name()}
|
||||
Path() string
|
||||
|
||||
// Stack can be used to gain access to the "docker compose" stack.
|
||||
//
|
||||
// This should internally call
|
||||
Stack() Installable
|
||||
|
||||
// Context returns a new InstallationContext to be used during installation from the command line.
|
||||
// Typically this should just pass through the parent, but might perform other tasks.
|
||||
Context(parent InstallationContext) InstallationContext
|
||||
|
||||
// Base() returns a reference to a base component
|
||||
// This is implemented by an embedding on ComponentBase
|
||||
Base() *ComponentBase
|
||||
}
|
||||
|
||||
// ComponentBase implements base functionality for a component
|
||||
type ComponentBase struct {
|
||||
Dir string // Dir is the directory this component lives in
|
||||
|
||||
Config *config.Config // Config is the configuration of the underlying distillery
|
||||
}
|
||||
|
||||
// Base returns a reference to the ComponentBase
|
||||
func (cb *ComponentBase) Base() *ComponentBase {
|
||||
return cb
|
||||
}
|
||||
|
||||
// Path returns the path to this component
|
||||
func (cb ComponentBase) Path() string {
|
||||
return cb.Dir
|
||||
}
|
||||
|
||||
// Context passes through the parent context
|
||||
func (ComponentBase) Context(parent InstallationContext) InstallationContext {
|
||||
return parent
|
||||
}
|
||||
|
||||
// MakeStack registers the Installable as a stack
|
||||
func (cb ComponentBase) MakeStack(stack Installable) Installable {
|
||||
stack.Dir = cb.Dir
|
||||
return stack
|
||||
}
|
||||
9
internal/component/dis/dis.env
Normal file
9
internal/component/dis/dis.env
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
VIRTUAL_HOST=${VIRTUAL_HOST}
|
||||
|
||||
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
|
||||
|
||||
CONFIG_PATH=${CONFIG_PATH}
|
||||
DEPLOY_ROOT=${DEPLOY_ROOT}
|
||||
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
|
||||
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
|
||||
48
internal/component/dis/dis.go
Normal file
48
internal/component/dis/dis.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package dis
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
)
|
||||
|
||||
type Dis struct {
|
||||
component.ComponentBase
|
||||
|
||||
Executable string // path to the current executable
|
||||
}
|
||||
|
||||
func (dis Dis) Name() string {
|
||||
return "dis"
|
||||
}
|
||||
|
||||
//go:embed all:stack dis.env
|
||||
var resources embed.FS
|
||||
|
||||
func (dis Dis) Stack() component.Installable {
|
||||
return dis.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
EnvPath: "dis.env",
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"VIRTUAL_HOST": dis.Config.DefaultHost(),
|
||||
"LETSENCRYPT_HOST": dis.Config.DefaultSSLHost(),
|
||||
"LETSENCRYPT_EMAIL": dis.Config.CertbotEmail,
|
||||
|
||||
"CONFIG_PATH": dis.Config.ConfigPath,
|
||||
"DEPLOY_ROOT": dis.Config.DeployRoot,
|
||||
|
||||
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
|
||||
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
|
||||
},
|
||||
CopyContextFiles: []string{core.Executable},
|
||||
})
|
||||
}
|
||||
|
||||
func (dis Dis) Context(parent component.InstallationContext) component.InstallationContext {
|
||||
return component.InstallationContext{
|
||||
core.Executable: dis.Executable,
|
||||
}
|
||||
}
|
||||
5
internal/component/dis/stack/Dockerfile
Normal file
5
internal/component/dis/stack/Dockerfile
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
FROM docker.io/library/alpine
|
||||
|
||||
COPY wdcli /wdcli
|
||||
EXPOSE 8888
|
||||
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","dis_server","--bind","0.0.0.0:8888"]
|
||||
28
internal/component/dis/stack/docker-compose.yml
Normal file
28
internal/component/dis/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
wdresolve:
|
||||
build: .
|
||||
restart: always
|
||||
environment:
|
||||
# port and hostname for this image to use
|
||||
VIRTUAL_HOST: ${VIRTUAL_HOST}
|
||||
VIRTUAL_PORT: 8888
|
||||
VIRTUAL_PATH: /dis/
|
||||
|
||||
CONFIG_PATH: ${CONFIG_PATH}
|
||||
|
||||
# optional letsencrypt email
|
||||
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
|
||||
|
||||
volumes:
|
||||
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
|
||||
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
|
||||
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
|
||||
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
118
internal/component/installable.go
Normal file
118
internal/component/installable.go
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
package component
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// TODO: Move this package into components
|
||||
|
||||
// Installable represents a Stack that can be automatically installed from a set of resources
|
||||
// See the [Install] method.
|
||||
type Installable struct {
|
||||
Stack
|
||||
|
||||
// Installable enabled installing several resources from a (potentially embedded) filesystem.
|
||||
//
|
||||
// The Resources holds these, with appropriate resources specified below.
|
||||
// These all refer to paths within the Resource filesystem.
|
||||
Resources fs.FS
|
||||
ContextPath string // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
|
||||
EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate].
|
||||
EnvContext map[string]string // context when instantiating the '.env' template
|
||||
|
||||
CopyContextFiles []string // Files to copy from the installation context
|
||||
|
||||
MakeDirsPerm fs.FileMode // permission for diretories, defaults to fs.ModeDir
|
||||
MakeDirs []string // directories to ensure that exist
|
||||
|
||||
TouchFiles []string // Files to 'touch', i.e. ensure that exist; guaranteed to be run after MakeDirs
|
||||
}
|
||||
|
||||
// InstallationContext is a context to install data in
|
||||
type InstallationContext map[string]string
|
||||
|
||||
// Install installs or updates this stack into the directory specified by stack.Stack().
|
||||
//
|
||||
// Installation is non-interactive, but will provide debugging output onto io.
|
||||
// InstallationContext
|
||||
func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
|
||||
if is.ContextPath != "" {
|
||||
// setup the base files
|
||||
if err := unpack.InstallDir(
|
||||
is.Dir,
|
||||
is.ContextPath,
|
||||
is.Resources,
|
||||
func(dst, src string) {
|
||||
io.Printf("[install] %s\n", dst)
|
||||
},
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// configure .env
|
||||
envDest := filepath.Join(is.Dir, ".env")
|
||||
if is.EnvPath != "" && is.EnvContext != nil {
|
||||
io.Printf("[config] %s\n", envDest)
|
||||
if err := unpack.InstallTemplate(
|
||||
envDest,
|
||||
is.EnvContext,
|
||||
is.EnvPath,
|
||||
is.Resources,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// make sure that certain dirs exist
|
||||
for _, name := range is.MakeDirs {
|
||||
// find the destination!
|
||||
dst := filepath.Join(is.Dir, name)
|
||||
|
||||
io.Printf("[make] %s\n", dst)
|
||||
if is.MakeDirsPerm == fs.FileMode(0) {
|
||||
is.MakeDirsPerm = fs.ModeDir
|
||||
}
|
||||
if err := os.MkdirAll(dst, is.MakeDirsPerm); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// copy files from the context!
|
||||
for _, name := range is.CopyContextFiles {
|
||||
// find the source!
|
||||
src, ok := context[name]
|
||||
if !ok {
|
||||
return errors.Errorf("Missing file from context: %s", src)
|
||||
}
|
||||
|
||||
// find the destination!
|
||||
dst := filepath.Join(is.Dir, name)
|
||||
|
||||
// copy over file from context
|
||||
io.Printf("[copy] %s (from %s)\n", dst, src)
|
||||
if err := fsx.CopyFile(dst, src); err != nil {
|
||||
return errors.Wrapf(err, "Unable to copy file %s", src)
|
||||
}
|
||||
}
|
||||
|
||||
// make sure that certain files exist
|
||||
for _, name := range is.TouchFiles {
|
||||
// find the destination!
|
||||
dst := filepath.Join(is.Dir, name)
|
||||
|
||||
io.Printf("[touch] %s\n", dst)
|
||||
if err := fsx.Touch(dst); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
10
internal/component/resolver/resolver.env
Normal file
10
internal/component/resolver/resolver.env
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
VIRTUAL_HOST=${VIRTUAL_HOST}
|
||||
|
||||
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
|
||||
|
||||
CONFIG_PATH=${CONFIG_PATH}
|
||||
DEPLOY_ROOT=${DEPLOY_ROOT}
|
||||
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
|
||||
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
|
||||
RESOLVER_CONFIG=${RESOLVER_CONFIG}
|
||||
109
internal/component/resolver/resolver.go
Normal file
109
internal/component/resolver/resolver.go
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
package resolver
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
|
||||
"github.com/FAU-CDI/wdresolve"
|
||||
"github.com/FAU-CDI/wdresolve/resolvers"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// TODO: Add a 'self-server' concept!
|
||||
|
||||
type Resolver struct {
|
||||
component.ComponentBase
|
||||
|
||||
ConfigName string // the name to the config file
|
||||
Executable string // path to the current executable
|
||||
}
|
||||
|
||||
func (Resolver) Name() string {
|
||||
return "resolver"
|
||||
}
|
||||
|
||||
func (resolver Resolver) ConfigPath() string {
|
||||
return filepath.Join(resolver.Dir, resolver.ConfigName)
|
||||
}
|
||||
|
||||
//go:embed all:stack resolver.env
|
||||
var resources embed.FS
|
||||
|
||||
func (resolver Resolver) Stack() component.Installable {
|
||||
return resolver.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
EnvPath: "resolver.env",
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"VIRTUAL_HOST": resolver.Config.DefaultHost(),
|
||||
"LETSENCRYPT_HOST": resolver.Config.DefaultSSLHost(),
|
||||
"LETSENCRYPT_EMAIL": resolver.Config.CertbotEmail,
|
||||
|
||||
"CONFIG_PATH": resolver.Config.ConfigPath,
|
||||
"DEPLOY_ROOT": resolver.Config.DeployRoot,
|
||||
|
||||
"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.Config.GlobalAuthorizedKeysFile,
|
||||
"SELF_OVERRIDES_FILE": resolver.Config.SelfOverridesFile,
|
||||
"RESOLVER_CONFIG": resolver.ConfigPath(),
|
||||
},
|
||||
TouchFiles: []string{resolver.ConfigName},
|
||||
CopyContextFiles: []string{core.Executable},
|
||||
})
|
||||
}
|
||||
|
||||
func (resolver Resolver) Context(parent component.InstallationContext) component.InstallationContext {
|
||||
return component.InstallationContext{
|
||||
core.Executable: resolver.Executable,
|
||||
}
|
||||
}
|
||||
|
||||
func (resolver Resolver) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
|
||||
p.TrustXForwardedProto = true
|
||||
|
||||
fallback := &resolvers.Regexp{
|
||||
Data: map[string]string{},
|
||||
}
|
||||
|
||||
// handle the default domain name!
|
||||
domainName := resolver.Config.DefaultDomain
|
||||
if domainName != "" {
|
||||
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
|
||||
io.Printf("registering default domain %s\n", domainName)
|
||||
}
|
||||
|
||||
// handle the extra domains!
|
||||
for _, domain := range resolver.Config.SelfExtraDomains {
|
||||
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
|
||||
io.Printf("registering legacy domain %s\n", domain)
|
||||
}
|
||||
|
||||
// open the prefix file
|
||||
prefixFile := resolver.ConfigPath()
|
||||
fs, err := os.Open(prefixFile)
|
||||
io.Println("loading prefixes from ", prefixFile)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
defer fs.Close()
|
||||
|
||||
// read the prefixes
|
||||
// TODO: Do we want to load these without a file?
|
||||
prefixes, err := resolvers.ReadPrefixes(fs)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
|
||||
// and use that as the resolver!
|
||||
p.Resolver = resolvers.InOrder{
|
||||
prefixes,
|
||||
fallback,
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
5
internal/component/resolver/stack/Dockerfile
Normal file
5
internal/component/resolver/stack/Dockerfile
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
FROM docker.io/library/alpine
|
||||
|
||||
COPY wdcli /wdcli
|
||||
EXPOSE 8888
|
||||
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]
|
||||
29
internal/component/resolver/stack/docker-compose.yml
Normal file
29
internal/component/resolver/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
wdresolve:
|
||||
build: .
|
||||
restart: always
|
||||
environment:
|
||||
# port and hostname for this image to use
|
||||
VIRTUAL_HOST: ${VIRTUAL_HOST}
|
||||
VIRTUAL_PORT: 8888
|
||||
VIRTUAL_PATH: /go/
|
||||
|
||||
CONFIG_PATH: ${CONFIG_PATH}
|
||||
|
||||
# optional letsencrypt email
|
||||
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
|
||||
|
||||
volumes:
|
||||
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
|
||||
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
|
||||
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
|
||||
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
|
||||
- "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
7
internal/component/self/self.env
Normal file
7
internal/component/self/self.env
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
VIRTUAL_HOST=${VIRTUAL_HOST}
|
||||
|
||||
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
|
||||
|
||||
TARGET=${TARGET}
|
||||
OVERRIDES_FILE=${OVERRIDES_FILE}
|
||||
42
internal/component/self/self.go
Normal file
42
internal/component/self/self.go
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
package self
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
type Self struct {
|
||||
component.ComponentBase
|
||||
}
|
||||
|
||||
func (Self) Name() string {
|
||||
return "self"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
//go:embed self.env
|
||||
var resources embed.FS
|
||||
|
||||
func (self Self) Stack() component.Installable {
|
||||
// TODO: Move me into config!
|
||||
TARGET := "https://github.com/FAU-CDI/wisski-distillery"
|
||||
if self.Config.SelfRedirect != nil { // TODO: move to config!
|
||||
TARGET = self.Config.SelfRedirect.String()
|
||||
}
|
||||
|
||||
return self.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
|
||||
ContextPath: "stack",
|
||||
EnvPath: "self.env",
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"VIRTUAL_HOST": self.Config.DefaultHost(),
|
||||
"LETSENCRYPT_HOST": self.Config.DefaultSSLHost(),
|
||||
"LETSENCRYPT_EMAIL": self.Config.CertbotEmail,
|
||||
"TARGET": TARGET,
|
||||
"OVERRIDES_FILE": self.Config.SelfOverridesFile,
|
||||
},
|
||||
})
|
||||
}
|
||||
28
internal/component/self/stack/docker-compose.yml
Normal file
28
internal/component/self/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
tr:
|
||||
image: ghcr.io/tkw1536/tr:latest
|
||||
restart: always
|
||||
volumes:
|
||||
- "${OVERRIDES_FILE}:/overrides.json:ro"
|
||||
environment:
|
||||
# port and hostname for this image to use
|
||||
VIRTUAL_HOST: ${VIRTUAL_HOST}
|
||||
VIRTUAL_PORT: 8080
|
||||
VIRTUAL_PATH: /
|
||||
|
||||
# optional letsencrypt email
|
||||
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
|
||||
|
||||
# the overrides file
|
||||
OVERRIDES: /overrides.json
|
||||
|
||||
# where to redirect to
|
||||
TARGET: ${TARGET}
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
217
internal/component/sql/database.go
Normal file
217
internal/component/sql/database.go
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
package sql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/sqle"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// sqlOpen opens a new sql connection to the provided database using the administrative credentials
|
||||
func (sql SQL) openDatabase(database string, config *gorm.Config) (*gorm.DB, error) {
|
||||
cfg := mysql.Config{
|
||||
DSN: fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8&parseTime=True&loc=Local", sql.Config.MysqlAdminUser, sql.Config.MysqlAdminPassword, sql.ServerURL, database),
|
||||
DefaultStringSize: 256,
|
||||
}
|
||||
|
||||
db, err := gorm.Open(mysql.New(cfg), config)
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
|
||||
gdb, err := db.DB()
|
||||
if err != nil {
|
||||
return db, err
|
||||
}
|
||||
gdb.SetMaxIdleConns(0)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// OpenBookkeeping opens a connection to the bookkeeping database
|
||||
func (sql SQL) OpenBookkeeping(silent bool) (*gorm.DB, error) {
|
||||
|
||||
config := &gorm.Config{}
|
||||
if silent {
|
||||
config.Logger = logger.Default.LogMode(logger.Silent)
|
||||
}
|
||||
|
||||
// open the database
|
||||
db, err := sql.openDatabase(sql.Config.DistilleryBookkeepingDatabase, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// load the table
|
||||
table := db.Table(sql.Config.DistilleryBookkeepingTable)
|
||||
if table.Error != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return table, nil
|
||||
}
|
||||
|
||||
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
|
||||
|
||||
// Backup makes a backup of the sql database into dest.
|
||||
func (sql SQL) Backup(io stream.IOStream, dest io.Writer, database string) error {
|
||||
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
|
||||
|
||||
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--databases", database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errSQLBackup
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackupAll makes a backup of all sql databases
|
||||
func (sql SQL) BackupAll(io stream.IOStream, dest io.Writer) error {
|
||||
io = stream.NewIOStream(dest, io.Stderr, nil, 0)
|
||||
|
||||
code, err := sql.Stack().Exec(io, "sql", "mysqldump", "--all-databases")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errSQLBackup
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// OpenShell executes a mysql shell command
|
||||
func (sql SQL) OpenShell(io stream.IOStream, argv ...string) (int, error) {
|
||||
return sql.Stack().Exec(io, "sql", "mysql", argv...)
|
||||
}
|
||||
|
||||
// WaitShell waits for the sql database to be reachable via a docker-compose shell
|
||||
func (sql SQL) WaitShell() error {
|
||||
n := stream.FromNil()
|
||||
return wait.Wait(func() bool {
|
||||
code, err := sql.OpenShell(n, "-e", "show databases;")
|
||||
return err == nil && code == 0
|
||||
}, sql.PollInterval, sql.PollContext)
|
||||
}
|
||||
|
||||
// Wait waits for a connection to the bookkeeping table to suceed
|
||||
func (sql SQL) Wait() error {
|
||||
return wait.Wait(func() bool {
|
||||
_, err := sql.OpenBookkeeping(true)
|
||||
return err == nil
|
||||
}, sql.PollInterval, sql.PollContext)
|
||||
}
|
||||
|
||||
var errInvalidDatabaseName = errors.New("SQLProvision: Invalid database name")
|
||||
|
||||
func (sql SQL) Query(query string, args ...interface{}) bool {
|
||||
raw := sqle.Format(query, args...)
|
||||
code, err := sql.OpenShell(stream.FromNil(), "-e", raw)
|
||||
return err == nil && code == 0
|
||||
}
|
||||
|
||||
// SQLProvision provisions a new sql database and user
|
||||
func (sql SQL) Provision(name, user, password string) error {
|
||||
// wait for the database
|
||||
if err := sql.WaitShell(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// it's not a safe database name!
|
||||
if !sqle.IsSafeDatabaseName(name) {
|
||||
return errInvalidDatabaseName
|
||||
}
|
||||
|
||||
// create the database and user!
|
||||
if !sql.Query("CREATE DATABASE `"+name+"`; CREATE USER ?@`%` IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON `"+name+"`.* TO ?@`%`; FLUSH PRIVILEGES;", user, password, user) {
|
||||
return errors.New("SQLProvision: Failed to create user")
|
||||
}
|
||||
|
||||
// and done!
|
||||
return nil
|
||||
}
|
||||
|
||||
var errSQLPurgeUser = errors.New("unable to delete user")
|
||||
|
||||
// SQLPurgeUser deletes the specified user from the database
|
||||
func (sql SQL) PurgeUser(user string) error {
|
||||
if !sql.Query("DROP USER IF EXISTS ?@`%`; FLUSH PRIVILEGES; ", user) {
|
||||
return errSQLPurgeUser
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var errSQLPurgeDB = errors.New("unable to drop database")
|
||||
|
||||
// SQLPurgeDatabase deletes the specified db from the database
|
||||
func (sql SQL) PurgeDatabase(db string) error {
|
||||
if !sqle.IsSafeDatabaseName(db) {
|
||||
return errSQLPurgeDB
|
||||
}
|
||||
if !sql.Query("DROP DATABASE IF EXISTS `" + db + "`") {
|
||||
return errSQLPurgeDB
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errSQLUnableToCreateUser = errors.New("unable to create administrative user")
|
||||
var errSQLUnsafeDatabaseName = errors.New("Bookkeeping database has an unsafe name")
|
||||
var errSQLUnableToCreate = errors.New("unable to create bookkeeping database")
|
||||
|
||||
// Bootstrap bootstraps the SQL database, and makes sure that the bookkeeping table is up-to-date
|
||||
func (sql SQL) Bootstrap(io stream.IOStream) error {
|
||||
if err := sql.WaitShell(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the admin user
|
||||
logging.LogMessage(io, "Creating administrative user")
|
||||
{
|
||||
username := sql.Config.MysqlAdminUser
|
||||
password := sql.Config.MysqlAdminPassword
|
||||
if !sql.Query("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?; GRANT ALL PRIVILEGES ON *.* TO ?@`%` WITH GRANT OPTION; FLUSH PRIVILEGES;", username, password, username) {
|
||||
return errSQLUnableToCreateUser
|
||||
}
|
||||
}
|
||||
|
||||
// create the admin user
|
||||
logging.LogMessage(io, "Creating sql database")
|
||||
{
|
||||
if !sqle.IsSafeDatabaseName(sql.Config.DistilleryBookkeepingDatabase) {
|
||||
return errSQLUnsafeDatabaseName
|
||||
}
|
||||
createDBSQL := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`;", sql.Config.DistilleryBookkeepingDatabase)
|
||||
if !sql.Query(createDBSQL) {
|
||||
return errSQLUnableToCreate
|
||||
}
|
||||
}
|
||||
|
||||
// wait for the database to come up
|
||||
logging.LogMessage(io, "Waiting for database update to be complete")
|
||||
sql.Wait()
|
||||
|
||||
// open the database
|
||||
logging.LogMessage(io, "Migrating bookkeeping table")
|
||||
{
|
||||
db, err := sql.OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to access bookkeeping table: %s", err)
|
||||
}
|
||||
|
||||
if err := db.AutoMigrate(&bookkeeping.Instance{}); err != nil {
|
||||
return fmt.Errorf("unable to migrate bookkeeping table: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
38
internal/component/sql/sql.go
Normal file
38
internal/component/sql/sql.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"io/fs"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
type SQL struct {
|
||||
component.ComponentBase
|
||||
|
||||
ServerURL string // upstream server url
|
||||
|
||||
PollContext context.Context // context to abort polling with
|
||||
PollInterval time.Duration // duration to wait for during wait
|
||||
}
|
||||
|
||||
func (SQL) Name() string {
|
||||
return "sql"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
var resources embed.FS
|
||||
|
||||
func (ssh SQL) Stack() component.Installable {
|
||||
return ssh.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
|
||||
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
|
||||
MakeDirs: []string{
|
||||
"data",
|
||||
},
|
||||
})
|
||||
}
|
||||
35
internal/component/sql/stack/docker-compose.yml
Normal file
35
internal/component/sql/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
sql:
|
||||
image: mariadb
|
||||
volumes:
|
||||
- "./data/:/var/lib/mysql"
|
||||
ports:
|
||||
- 127.0.0.1:3306:3306
|
||||
environment:
|
||||
# This combination of environment variables will configure a passwordless root user
|
||||
# that can only connect to the container from 'localhost'.
|
||||
# This means we can only connect using 'docker-compose exec sql mysql -C '...' '.
|
||||
- "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
|
||||
- "MYSQL_ROOT_HOST=localhost"
|
||||
restart: always
|
||||
phpmyadmin:
|
||||
image: phpmyadmin/phpmyadmin
|
||||
environment:
|
||||
- "PMA_HOST=sql"
|
||||
- "HIDE_PHP_VERSION=true"
|
||||
- "UPLOAD_LIMIT=100M"
|
||||
# phpmyadmin running on localhost:8080 so that we can easily access the system graphically.
|
||||
# By default no admin account is created, so initial shell access to make one is needed.
|
||||
ports:
|
||||
- 127.0.0.1:8080:80
|
||||
depends_on:
|
||||
- sql
|
||||
restart: always
|
||||
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
25
internal/component/ssh/ssh.go
Normal file
25
internal/component/ssh/ssh.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package ssh
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
type SSH struct {
|
||||
component.ComponentBase
|
||||
}
|
||||
|
||||
func (SSH) Name() string {
|
||||
return "ssh"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
var resources embed.FS
|
||||
|
||||
func (ssh SSH) Stack() component.Installable {
|
||||
return ssh.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
})
|
||||
}
|
||||
17
internal/component/ssh/stack/docker-compose.yml
Normal file
17
internal/component/ssh/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
ssh:
|
||||
image: ghcr.io/tkw1536/dockersshd:latest
|
||||
command: -hostkey /keys/hostkey -shell /user_shell.sh -keylabel eu.wiss-ki.barrel.authfile -userlabel eu.wiss-ki.barrel.slug -L triplestore:7200 -L phpmyadmin:80 -L sql:3306
|
||||
ports:
|
||||
- "2222:2222"
|
||||
volumes:
|
||||
- './data/keys:/keys'
|
||||
- '/var/run/docker.sock:/var/run/docker.sock:ro'
|
||||
restart: always
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
150
internal/component/stack.go
Normal file
150
internal/component/stack.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
// Package stack implements a docker compose stack
|
||||
package component
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/execx"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// Stack represents a 'docker compose' stack living in a specific directory
|
||||
//
|
||||
// NOTE(twiesing): In the current implementation this requires a 'docker' executable on the system.
|
||||
// This executable must be capable of the 'docker compose' command.
|
||||
// In the future the idea is to replace this with a native docker compose client.
|
||||
type Stack struct {
|
||||
Dir string // Directory this Stack is located in
|
||||
|
||||
DockerExecutable string // Path to the native docker executable to use
|
||||
}
|
||||
|
||||
var errStackUpdatePull = errors.New("Stack.Update: Pull returned non-zero exit code")
|
||||
var errStackUpdateBuild = errors.New("Stack.Update: Build returned non-zero exit code")
|
||||
|
||||
// Update pulls, builds, and then optionally starts this stack.
|
||||
// This does not have a direct 'docker compose' shell equivalent.
|
||||
//
|
||||
// See also Up.
|
||||
func (ds Stack) Update(io stream.IOStream, start bool) error {
|
||||
{
|
||||
code, err := ds.compose(io, "pull")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errStackUpdatePull
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
code, err := ds.compose(io, "build", "--pull")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errStackUpdateBuild
|
||||
}
|
||||
}
|
||||
if start {
|
||||
return ds.Up(io)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errStackUp = errors.New("Stack.Up: Up returned non-zero exit code")
|
||||
|
||||
// Up creates and starts the containers in this Stack.
|
||||
// It is equivalent to 'docker compose up --remove-orphans --detach' on the shell.
|
||||
func (ds Stack) Up(io stream.IOStream) error {
|
||||
code, err := ds.compose(io, "up", "--remove-orphans", "--detach")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errStackUp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exec executes an executable in the provided running service.
|
||||
// It is equivalent to 'docker compose exec $service $executable $args...'.
|
||||
//
|
||||
// It returns the exit code of the process.
|
||||
func (ds Stack) Exec(io stream.IOStream, service, executable string, args ...string) (int, error) {
|
||||
compose := []string{"exec"}
|
||||
if io.StdinIsATerminal() {
|
||||
compose = append(compose, "-ti")
|
||||
}
|
||||
compose = append(compose, service)
|
||||
compose = append(compose, executable)
|
||||
compose = append(compose, args...)
|
||||
return ds.compose(io, compose...)
|
||||
}
|
||||
|
||||
// Run executes the provided service with the given executable.
|
||||
// It is equivalent to 'docker compose run [--rm] $service $executable $args...'.
|
||||
//
|
||||
// It returns the exit code of the process.
|
||||
func (ds Stack) Run(io stream.IOStream, autoRemove bool, service, command string, args ...string) (int, error) {
|
||||
compose := []string{"run"}
|
||||
if autoRemove {
|
||||
compose = append(compose, "--rm")
|
||||
}
|
||||
if !io.StdinIsATerminal() {
|
||||
compose = append(compose, "-T")
|
||||
}
|
||||
compose = append(compose, service, command)
|
||||
compose = append(compose, args...)
|
||||
|
||||
code, err := ds.compose(io, compose...)
|
||||
if err != nil {
|
||||
return execx.ExecCommandError, nil
|
||||
}
|
||||
return code, nil
|
||||
}
|
||||
|
||||
var errStackRestart = errors.New("Stack.Restart: Restart returned non-zero exit code")
|
||||
|
||||
// Restart restarts all containers in this Stack.
|
||||
// It is equivalent to 'docker compose restart' on the shell.
|
||||
func (ds Stack) Restart(io stream.IOStream) error {
|
||||
code, err := ds.compose(io, "restart")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errStackRestart
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errStackDown = errors.New("Stack.Down: Down returned non-zero exit code")
|
||||
|
||||
// Down stops and removes all containers in this Stack.
|
||||
// It is equivalent to 'docker compose down -v' on the shell.
|
||||
func (ds Stack) Down(io stream.IOStream) error {
|
||||
code, err := ds.compose(io, "down", "-v")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errStackDown
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// compose executes a 'docker compose' command on this stack.
|
||||
//
|
||||
// NOTE(twiesing): Check if this can be replaced by an internal call to libcompose.
|
||||
// But probably not.
|
||||
func (ds Stack) compose(io stream.IOStream, args ...string) (int, error) {
|
||||
if ds.DockerExecutable == "" {
|
||||
var err error
|
||||
ds.DockerExecutable, err = execx.LookPathAbs("docker")
|
||||
if err != nil {
|
||||
return execx.ExecCommandError, err
|
||||
}
|
||||
}
|
||||
return execx.Exec(io, ds.Dir, ds.DockerExecutable, append([]string{"compose"}, args...)...), nil
|
||||
}
|
||||
57
internal/component/triplestore/create-repo.ttl
Normal file
57
internal/component/triplestore/create-repo.ttl
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# This file is used to initialize a new GraphDB repository.
|
||||
# In this file the variables ${GRAPHDB_REPO} and ${INSTANCE_DOMAIN} will be replaced.
|
||||
# All other variables will be left untouched.
|
||||
|
||||
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
|
||||
@prefix rep: <http://www.openrdf.org/config/repository#>.
|
||||
@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
|
||||
@prefix sail: <http://www.openrdf.org/config/sail#>.
|
||||
@prefix owlim: <http://www.ontotext.com/trree/owlim#>.
|
||||
|
||||
[] a rep:Repository ;
|
||||
rep:repositoryID "${GRAPHDB_REPO}" ;
|
||||
rdfs:label "${INSTANCE_DOMAIN}" ;
|
||||
rep:repositoryImpl [
|
||||
rep:repositoryType "graphdb:SailRepository" ;
|
||||
sr:sailImpl [
|
||||
sail:sailType "graphdb:Sail" ;
|
||||
|
||||
owlim:owlim-license "" ;
|
||||
|
||||
owlim:base-URL "http://${INSTANCE_DOMAIN}/" ;
|
||||
owlim:defaultNS "" ;
|
||||
owlim:entity-index-size "10000000" ;
|
||||
owlim:entity-id-size "32" ;
|
||||
owlim:imports "" ;
|
||||
owlim:repository-type "file-repository" ;
|
||||
owlim:ruleset "empty" ;
|
||||
owlim:storage-folder "storage" ;
|
||||
|
||||
owlim:enable-context-index "false" ;
|
||||
owlim:cache-memory "80m" ;
|
||||
owlim:tuple-index-memory "80m" ;
|
||||
|
||||
owlim:enablePredicateList "false" ;
|
||||
owlim:predicate-memory "0%" ;
|
||||
|
||||
owlim:fts-memory "0%" ;
|
||||
owlim:ftsIndexPolicy "never" ;
|
||||
owlim:ftsLiteralsOnly "true" ;
|
||||
|
||||
owlim:in-memory-literal-properties "false" ;
|
||||
owlim:enable-literal-index "true" ;
|
||||
owlim:index-compression-ratio "-1" ;
|
||||
|
||||
owlim:check-for-inconsistencies "false" ;
|
||||
owlim:disable-sameAs "false" ;
|
||||
owlim:enable-optimization "true" ;
|
||||
owlim:transaction-mode "safe" ;
|
||||
owlim:transaction-isolation "true" ;
|
||||
owlim:query-timeout "0" ;
|
||||
owlim:query-limit-results "0" ;
|
||||
owlim:throw-QueryEvaluationException-on-timeout "false" ;
|
||||
owlim:useShutdownHooks "true" ;
|
||||
owlim:read-only "false" ;
|
||||
owlim:nonInterpretablePredicates "http://www.w3.org/2000/01/rdf-schema#label;http://www.w3.org/1999/02/22-rdf-syntax-ns#type;http://www.ontotext.com/owlim/ces#gazetteerConfig;http://www.ontotext.com/owlim/ces#metadataConfig" ;
|
||||
]
|
||||
].
|
||||
247
internal/component/triplestore/database.go
Normal file
247
internal/component/triplestore/database.go
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
package triplestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/wait"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
type TriplestoreUserPayload struct {
|
||||
Password string `json:"password"`
|
||||
AppSettings TriplestoreUserAppSettings `json:"appSettings"`
|
||||
GrantedAuthorities []string `json:"grantedAuthorities"`
|
||||
}
|
||||
type TriplestoreUserAppSettings struct {
|
||||
DefaultInference bool `json:"DEFAULT_INFERENCE"`
|
||||
DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
|
||||
DefaultSameas bool `json:"DEFAULT_SAMEAS"`
|
||||
IgnoreSharedQueries bool `json:"IGNORE_SHARED_QUERIES"`
|
||||
ExecuteCount bool `json:"EXECUTE_COUNT"`
|
||||
}
|
||||
|
||||
// OpenRaw makes an http request to the triplestore api.
|
||||
//
|
||||
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
|
||||
// When bodyName is empty, simply marshal body as application/json
|
||||
func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
|
||||
var reader io.Reader
|
||||
|
||||
var contentType string
|
||||
|
||||
// for "PUT" and "POST" we setup a body
|
||||
if method == "PUT" || method == "POST" {
|
||||
if bodyName != "" {
|
||||
buffer := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(buffer)
|
||||
contentType = writer.FormDataContentType()
|
||||
|
||||
part, err := writer.CreateFormFile(bodyName, "filename.txt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
io.Copy(part, bytes.NewReader(body.([]byte)))
|
||||
writer.Close()
|
||||
reader = buffer
|
||||
} else {
|
||||
contentType = "application/json"
|
||||
mbytes, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader = bytes.NewReader(mbytes)
|
||||
}
|
||||
}
|
||||
|
||||
// create the request object
|
||||
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Setup configuration!
|
||||
if accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
|
||||
|
||||
// and send it
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
|
||||
// Wait waits for the connection to the Triplestore to succeed.
|
||||
// This is achieved using a polling strategy.
|
||||
func (ts Triplestore) Wait() error {
|
||||
return wait.Wait(func() bool {
|
||||
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
return true
|
||||
}, ts.PollInterval, ts.PollContext)
|
||||
}
|
||||
|
||||
// TriplestorePurgeUser deletes the specified user from the triplestore
|
||||
func (ts Triplestore) PurgeUser(user string) error {
|
||||
res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != http.StatusNoContent {
|
||||
return errors.Errorf("Delete returned code %d", res.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TriplestorePurgeRepo deletes the specified repo from the triplestore
|
||||
func (ts Triplestore) PurgeRepo(repo string) error {
|
||||
res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return errors.Errorf("Delete returned code %d", res.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
|
||||
|
||||
// TriplestoreBackup backs up the repository named repo into the writer dst.
|
||||
func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
|
||||
res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return 0, errTSBackupWrongStatusCode
|
||||
}
|
||||
defer res.Body.Close()
|
||||
return io.Copy(dst, res.Body)
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
URI string `json:"uri"`
|
||||
Type string `json:"type"`
|
||||
SesameType string `json:"sesameType"`
|
||||
Location string `json:"location"`
|
||||
Readable bool `json:"readable"`
|
||||
Writable bool `json:"writable"`
|
||||
Local bool `json:"local"`
|
||||
}
|
||||
|
||||
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
|
||||
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
err = json.NewDecoder(res.Body).Decode(&repos)
|
||||
return
|
||||
}
|
||||
|
||||
// TriplestoreBackup backs up every graphdb instance into dst
|
||||
func (ts Triplestore) BackupAll(dst string) error {
|
||||
// list all the repositories
|
||||
repos, err := ts.listRepositories()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the base directory
|
||||
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// iterate over all the repositories
|
||||
for _, repo := range repos {
|
||||
if rErr := (func(repo Repository) error {
|
||||
name := filepath.Join(dst, repo.ID+".nq")
|
||||
|
||||
dest, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dest.Close()
|
||||
|
||||
_, err = ts.Backup(dest, repo.ID)
|
||||
return err
|
||||
}(repo)); err == nil && rErr != nil {
|
||||
err = rErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
|
||||
|
||||
func (ts Triplestore) Bootstrap(io stream.IOStream) error {
|
||||
logging.LogMessage(io, "Waiting for Triplestore")
|
||||
if err := ts.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logging.LogMessage(io, "Resetting admin user password")
|
||||
{
|
||||
res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
|
||||
Password: ts.Config.TriplestoreAdminPassword,
|
||||
AppSettings: TriplestoreUserAppSettings{
|
||||
DefaultInference: true,
|
||||
DefaultVisGraphSchema: true,
|
||||
DefaultSameas: true,
|
||||
IgnoreSharedQueries: false,
|
||||
ExecuteCount: true,
|
||||
},
|
||||
GrantedAuthorities: []string{"ROLE_ADMIN"},
|
||||
}, "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create triplestore user: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
// we set the password => requests are unauthorized
|
||||
// so we still need to enable security (see below!)
|
||||
case http.StatusUnauthorized:
|
||||
// a password is needed => security is already enabled.
|
||||
// the password may or may not work, but that's a problem for later
|
||||
logging.LogMessage(io, "Security is already enabled")
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("failed to create triplestore user: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
logging.LogMessage(io, "Enabling Triplestore security")
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to enable triplestore security: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return errTriplestoreFailedSecurity
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
75
internal/component/triplestore/provision.go
Normal file
75
internal/component/triplestore/provision.go
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
package triplestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
|
||||
_ "embed"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
|
||||
"github.com/tkw1536/goprogram/exit"
|
||||
)
|
||||
|
||||
var errTripleStoreFailedRepository = exit.Error{
|
||||
Message: "Failed to create repository: %s",
|
||||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
//go:embed create-repo.ttl
|
||||
var createRepoTTL []byte
|
||||
|
||||
func (ts Triplestore) Provision(name, domain, user, password string) error {
|
||||
if err := ts.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// prepare the create repo request
|
||||
var createRepo bytes.Buffer
|
||||
err := unpack.WriteTemplate(&createRepo, map[string]string{
|
||||
"GRAPHDB_REPO": name,
|
||||
"INSTANCE_DOMAIN": domain,
|
||||
}, bytes.NewReader(createRepoTTL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do the create!
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo.Bytes(), "config", "")
|
||||
if err != nil {
|
||||
return errTripleStoreFailedRepository.WithMessageF(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusCreated {
|
||||
return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
|
||||
}
|
||||
}
|
||||
|
||||
// create the user and grant them access
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
|
||||
Password: password,
|
||||
AppSettings: TriplestoreUserAppSettings{
|
||||
DefaultInference: true,
|
||||
DefaultVisGraphSchema: true,
|
||||
DefaultSameas: true,
|
||||
IgnoreSharedQueries: false,
|
||||
ExecuteCount: true,
|
||||
},
|
||||
GrantedAuthorities: []string{
|
||||
"ROLE_USER",
|
||||
"READ_REPO_" + name,
|
||||
"WRITE_REPO_" + name,
|
||||
},
|
||||
}, "", "")
|
||||
if err != nil {
|
||||
return errTripleStoreFailedRepository.WithMessageF(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusCreated {
|
||||
return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
3
internal/component/triplestore/stack/.dockerignore
Normal file
3
internal/component/triplestore/stack/.dockerignore
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
*
|
||||
!*.zip
|
||||
!entrypoint.sh
|
||||
64
internal/component/triplestore/stack/Dockerfile
Normal file
64
internal/component/triplestore/stack/Dockerfile
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container.
|
||||
# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
|
||||
# but has been modified for performance and security.
|
||||
|
||||
# This image is intended to be built like:
|
||||
# docker build --build-arg graphdb_src=graphdb.zip .
|
||||
|
||||
# We first make a base image to base further builds on.
|
||||
# We don't use alpine here, as that uses significantly slower musl instead of glibc.
|
||||
FROM adoptopenjdk/openjdk11:debian-slim as base
|
||||
|
||||
# Create a user called graphdb
|
||||
RUN useradd -ms /bin/bash graphdb
|
||||
|
||||
# make a base images, to add the sources to.
|
||||
FROM base as sources
|
||||
|
||||
# install unzip
|
||||
RUN apt-get update && apt-get install -y unzip
|
||||
|
||||
# add the source file (by default graphdb.zip) to the image
|
||||
ARG src=graphdb.zip
|
||||
ADD ${src} /graphdb.zip
|
||||
|
||||
# unpack it into a temporary directory
|
||||
RUN unzip "$src" -d "/unpack/"
|
||||
|
||||
# Move it into /opt/graphdb, and chown it to graphdb
|
||||
RUN mv "/unpack"/* /opt/graphdb
|
||||
RUN chown -R graphdb:graphdb /opt/graphdb
|
||||
|
||||
# finally make an image that will run
|
||||
FROM base as final
|
||||
|
||||
# add the entrypoint script
|
||||
ADD entrypoint.sh /entrypoint.sh
|
||||
|
||||
# copy over the sources
|
||||
COPY --from=sources /opt/graphdb /opt/graphdb
|
||||
|
||||
# set environment variables for graphdb_home and path
|
||||
ENV GRAPHDB_HOME=/opt/graphdb
|
||||
ENV PATH=$GRAPHDB_HOME/bin:$PATH
|
||||
|
||||
# Workaround for CVE-2021-44228
|
||||
# (not sure if we are vulnerable, but just because)
|
||||
ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
|
||||
|
||||
# expose a port
|
||||
EXPOSE 7200
|
||||
|
||||
# setup a healthcheck, that checks if the server is up.
|
||||
RUN apt-get update && apt-get install -y curl
|
||||
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
|
||||
|
||||
# Add volumes for data, work and logs as these might be accessible from the outside.
|
||||
# To add your own configuration, manually mount a config file into /opt/graphdb/work
|
||||
VOLUME /opt/graphdb/data
|
||||
VOLUME /opt/graphdb/work
|
||||
VOLUME /opt/graphdb/logs
|
||||
|
||||
# setup command and entrypoint
|
||||
CMD ["-Dgraphdb.home=/opt/graphdb"]
|
||||
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
|
||||
22
internal/component/triplestore/stack/docker-compose.yml
Normal file
22
internal/component/triplestore/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
triplestore:
|
||||
build: .
|
||||
ports:
|
||||
- "127.0.0.1:7200:7200"
|
||||
volumes:
|
||||
- './data/data:/opt/graphdb/data'
|
||||
- './data/work:/opt/graphdb/work'
|
||||
- './data/logs:/opt/graphdb/logs'
|
||||
command: "\"-Dgraphdb.home=/opt/graphdb -Ddefault.min.distinct.threshold=2G\""
|
||||
# Use 1GB of heap space
|
||||
environment:
|
||||
GDB_HEAP_SIZE: 16G
|
||||
|
||||
restart: always
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
13
internal/component/triplestore/stack/entrypoint.sh
Normal file
13
internal/component/triplestore/stack/entrypoint.sh
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Because we want to run graphdb as a limited user
|
||||
# we need to make sure that the volumes are writable.
|
||||
# Because of that, we 'chown'
|
||||
|
||||
chown graphdb:graphdb /opt/graphdb/data
|
||||
chown graphdb:graphdb /opt/graphdb/work
|
||||
chown graphdb:graphdb /opt/graphdb/logs
|
||||
|
||||
# switch to the graphdb user, and run graphdb
|
||||
su graphdb -c "/opt/graphdb/bin/graphdb $@"
|
||||
43
internal/component/triplestore/triplestore.go
Normal file
43
internal/component/triplestore/triplestore.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package triplestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
type Triplestore struct {
|
||||
component.ComponentBase
|
||||
|
||||
BaseURL string // upstream server url
|
||||
|
||||
PollContext context.Context // context to abort polling with
|
||||
PollInterval time.Duration // duration to wait for during wait
|
||||
}
|
||||
|
||||
func (Triplestore) Name() string {
|
||||
return "triplestore"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
var resources embed.FS
|
||||
|
||||
func (ts Triplestore) Stack() component.Installable {
|
||||
return ts.ComponentBase.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
|
||||
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
|
||||
|
||||
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
|
||||
MakeDirs: []string{
|
||||
filepath.Join("data", "data"),
|
||||
filepath.Join("data", "work"),
|
||||
filepath.Join("data", "logs"),
|
||||
},
|
||||
})
|
||||
}
|
||||
52
internal/component/web/stack/docker-compose.yml
Normal file
52
internal/component/web/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
nginx-proxy:
|
||||
image: ghcr.io/nginx-proxy/nginx-proxy:alpine
|
||||
environment:
|
||||
- DEFAULT_HOST=${DEFAULT_HOST}
|
||||
- HTTPS_METHOD=${HTTPS_METHOD}
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- "vhost:/etc/nginx/vhost.d"
|
||||
- "./global.conf:/etc/nginx/conf.d/global.conf:ro"
|
||||
- "./proxy.conf:/etc/nginx/proxy.conf:ro"
|
||||
- "htpasswd:/etc/nginx/htpasswd"
|
||||
- "html:/usr/share/nginx/html"
|
||||
- "/var/run/docker.sock:/tmp/docker.sock:ro"
|
||||
- "certs:/etc/nginx/certs"
|
||||
labels:
|
||||
com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: true
|
||||
restart: always
|
||||
networks:
|
||||
- default
|
||||
|
||||
letsencrypt-nginx-proxy-companion:
|
||||
image: docker.io/nginxproxy/acme-companion:latest
|
||||
volumes:
|
||||
- "/var/run/docker.sock:/var/run/docker.sock:ro"
|
||||
- "htpasswd:/etc/nginx/htpasswd"
|
||||
- "vhost:/etc/nginx/vhost.d"
|
||||
- "html:/usr/share/nginx/html"
|
||||
- "/var/run/docker.sock:/tmp/docker.sock:ro"
|
||||
- "certs:/etc/nginx/certs"
|
||||
- "acme:/etc/acme.sh"
|
||||
restart: always
|
||||
networks:
|
||||
- default
|
||||
depends_on:
|
||||
- nginx-proxy
|
||||
|
||||
volumes:
|
||||
acme:
|
||||
vhost:
|
||||
html:
|
||||
certs:
|
||||
htpasswd:
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
4
internal/component/web/stack/global.conf
Normal file
4
internal/component/web/stack/global.conf
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# Nginx Configuration File
|
||||
# These should match with distillery/resources/compose/barrel/conf/wisski.ini.
|
||||
|
||||
client_max_body_size 1000m;
|
||||
19
internal/component/web/stack/proxy.conf
Normal file
19
internal/component/web/stack/proxy.conf
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# HTTP 1.1 support
|
||||
proxy_http_version 1.1;
|
||||
proxy_buffering off;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $proxy_connection;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
|
||||
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
|
||||
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
|
||||
|
||||
# Mitigate httpoxy attack (see README for details)
|
||||
proxy_set_header Proxy "";
|
||||
|
||||
# Timeouts for the proxy connection - in sync with the appropriate max_execution time.
|
||||
proxy_connect_timeout 3000s;
|
||||
proxy_read_timeout 3000s;
|
||||
proxy_send_timeout 3000s;
|
||||
2
internal/component/web/web.env
Normal file
2
internal/component/web/web.env
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
DEFAULT_HOST=${DEFAULT_HOST}
|
||||
HTTPS_METHOD=${HTTPS_METHOD}
|
||||
38
internal/component/web/web.go
Normal file
38
internal/component/web/web.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package web
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
)
|
||||
|
||||
// Web implements the web component
|
||||
type Web struct {
|
||||
component.ComponentBase
|
||||
}
|
||||
|
||||
func (Web) Name() string {
|
||||
return "web"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
//go:embed web.env
|
||||
var resources embed.FS
|
||||
|
||||
func (web Web) Stack() component.Installable {
|
||||
HTTPS_METHOD := "nohttp"
|
||||
if web.Config.HTTPSEnabled() {
|
||||
HTTPS_METHOD = "redirect"
|
||||
}
|
||||
|
||||
return web.MakeStack(component.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
EnvPath: "web.env",
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"DEFAULT_HOST": web.Config.DefaultDomain,
|
||||
"HTTPS_METHOD": HTTPS_METHOD,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package config
|
||||
|
||||
import "github.com/FAU-CDI/wisski-distillery/internal/password"
|
||||
import "github.com/FAU-CDI/wisski-distillery/pkg/password"
|
||||
|
||||
// NewPassword returns a new password using the password settings from this configuration
|
||||
func (cfg Config) NewPassword() (string, error) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
|
|
|
|||
15
internal/core/bootstrap.go
Normal file
15
internal/core/bootstrap.go
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package core
|
||||
|
||||
import _ "embed"
|
||||
|
||||
// DefaultOverridesJSON contains a template for a new 'overrides.json' file
|
||||
//go:embed bootstrap/overrides.json
|
||||
var DefaultOverridesJSON []byte
|
||||
|
||||
// DefaultAuthorizedKeys contains a template for a new 'global_authorized_keys' file
|
||||
//go:embed bootstrap/global_authorized_keys
|
||||
var DefaultAuthorizedKeys []byte
|
||||
|
||||
// ConfigFileTemplate contains a template for a new configuration file
|
||||
//go:embed bootstrap/env
|
||||
var ConfigFileTemplate []byte
|
||||
63
internal/core/bootstrap/env
Normal file
63
internal/core/bootstrap/env
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
# Several docker-compose files are created to manage global services and the system itself.
|
||||
# On top of this all real-system space will be created under this directory.
|
||||
DEPLOY_ROOT=${DEPLOY_ROOT}
|
||||
|
||||
# Each created Drupal Instance corresponds to a single domain name.
|
||||
# These domain names should either be a complete domain name or a sub-domain of a default domain.
|
||||
# This setting configures the default domain-name to create subdomains of.
|
||||
DEFAULT_DOMAIN=${DEFAULT_DOMAIN}
|
||||
|
||||
# By default, the default domain redirects to the distillery repository.
|
||||
# If you want to change this, set an alternate domain name here.
|
||||
SELF_REDIRECT=
|
||||
|
||||
# By default, only the 'self' domain above is caught.
|
||||
# To catch additional domains, add them here (comma seperated)
|
||||
SELF_EXTRA_DOMAINS=
|
||||
|
||||
# You can override individual URLS in the homepage.
|
||||
# Do this by adding URLs (without trailing '/'s) into a JSON file
|
||||
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
|
||||
|
||||
# The system can support setting up certificate(s) automatically.
|
||||
# It can be enabled by setting an email for certbot certificates.
|
||||
# This email address can be configured here.
|
||||
CERTBOT_EMAIL=
|
||||
|
||||
# The maximum age (in days) for backups to be kept.
|
||||
# Backups older than this will be removed when a new backup is made.
|
||||
MAX_BACKUP_AGE=30
|
||||
|
||||
|
||||
# Each Drupal instance requires a corresponding system user, database users and databases.
|
||||
# These are also set by the appropriate domain name.
|
||||
# To differentiate them from other users of the system, these names can be prefixed.
|
||||
# The prefix to use can be configured here.
|
||||
# When changing these please consider that no system user may exist that has the same name as a mysql user.
|
||||
# This is a MariaDB restriction.
|
||||
MYSQL_USER_PREFIX=mysql-factory-
|
||||
MYSQL_DATABASE_PREFIX=mysql-factory-
|
||||
GRAPHDB_USER_PREFIX=graphdb-factory-
|
||||
GRAPHDB_REPO_PREFIX=graphdb-factory-
|
||||
|
||||
# In addition to the filesystem the WissKI distillery requires a single SQL table.
|
||||
# It uses this database to store a list of installed things
|
||||
DISTILLERY_BOOKKEEPING_DATABASE=distillery
|
||||
DISTILLERY_BOOKKEEPING_TABLE=distillery
|
||||
|
||||
|
||||
# Various components use password-based-authentication.
|
||||
# These passwords are generated automatically.
|
||||
# This variable can be used to determine their length.
|
||||
PASSWORD_LENGTH=64
|
||||
|
||||
# A file to be used for global authorized_keys for the ssh server.
|
||||
GLOBAL_AUTHORIZED_KEYS_FILE=${AUTHORIZED_KEYS_FILE}
|
||||
|
||||
# The admin user and password of the GraphDB interface, to be used for queries
|
||||
GRAPHDB_ADMIN_USER=${GRAPHDB_ADMIN_USER}
|
||||
GRAPHDB_ADMIN_PASSWORD=${GRAPHDB_ADMIN_PASSWORD}
|
||||
|
||||
# The admin password to use for access to mysql
|
||||
MYSQL_ADMIN_USER=${MYSQL_ADMIN_USER}
|
||||
MYSQL_ADMIN_PASSWORD=${MYSQL_ADMIN_PASSWORD}
|
||||
2
internal/core/bootstrap/global_authorized_keys
Normal file
2
internal/core/bootstrap/global_authorized_keys
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# This file contains authorized_keys files valid for every repository in the distillery
|
||||
# The syntax of this file is easy, one key per line, empty lines or those starting with '#' are ignored
|
||||
1
internal/core/bootstrap/overrides.json
Normal file
1
internal/core/bootstrap/overrides.json
Normal file
|
|
@ -0,0 +1 @@
|
|||
{}
|
||||
14
internal/core/core.go
Normal file
14
internal/core/core.go
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
// Package core implements the core of the WissKI Distillery and the wdcli executable.
|
||||
// It does not depend on any other packages.
|
||||
package core
|
||||
|
||||
// BaseDirectoryDefault is the default deploy directory to load the distillery from.
|
||||
const BaseDirectoryDefault = "/var/www/deploy"
|
||||
|
||||
// Executable is the name of the 'wdcli' executable.
|
||||
// It should be located inside the deployment directory.
|
||||
const Executable = "wdcli"
|
||||
|
||||
// Config file is the name of the config file.
|
||||
// It should be located inside the deployment directory.
|
||||
const ConfigFile = ".env"
|
||||
8
internal/core/flags.go
Normal file
8
internal/core/flags.go
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package core
|
||||
|
||||
// Flags are global flags for the wdcli executable
|
||||
type Flags struct {
|
||||
ConfigPath string `short:"c" long:"config" description:"Path to distillery configuration file"`
|
||||
|
||||
InternalInDocker bool `long:"internal-in-docker" description:"Internal Flag to signal the shell that it is running inside a docker stack belonging to the distillery"`
|
||||
}
|
||||
71
internal/core/meta.go
Normal file
71
internal/core/meta.go
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MetaConfigFile is the path to a configuration file that contains the path to the last used wdcli executable.
|
||||
// It is expected to be in the current user's home directory.
|
||||
//
|
||||
// You probably want to use [MetaConfigPath] instead.
|
||||
//
|
||||
// It should contain the path to a deployment directory.
|
||||
const MetaConfigFile = "." + Executable
|
||||
|
||||
// MetaConfigPath returns the full path to the MetaConfigPath()
|
||||
func MetaConfigPath() (string, error) {
|
||||
// find the current user
|
||||
usr, err := user.Current()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(usr.HomeDir, MetaConfigFile), nil
|
||||
}
|
||||
|
||||
var errReadBaseDirectoryEmpty = errors.New("ReadBaseDirectory: Directory is empty")
|
||||
|
||||
// ReadBaseDirectory reads the base deployment directory from the environment.
|
||||
// Use [ParamsFromEnv] to initialize parameters completely.
|
||||
//
|
||||
// It does not perform any reading of files.
|
||||
func ReadBaseDirectory() (value string, err error) {
|
||||
// get the path!
|
||||
path, err := MetaConfigPath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// read the meta config file!
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// and trim the spaces!
|
||||
value = strings.TrimSpace(string(contents))
|
||||
|
||||
// check that it is actually set!
|
||||
if len(value) == 0 {
|
||||
return "", errReadBaseDirectoryEmpty
|
||||
}
|
||||
|
||||
// and return it!
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// WriteBaseDirectory writes the base directory to the environment, or returns an error
|
||||
func WriteBaseDirectory(dir string) error {
|
||||
// get the path!
|
||||
path, err := MetaConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// just put the directory inside it!
|
||||
return os.WriteFile(path, []byte(dir), fs.ModePerm)
|
||||
}
|
||||
31
internal/core/params.go
Normal file
31
internal/core/params.go
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Params are used to initialize the excutable.
|
||||
type Params struct {
|
||||
ConfigPath string // ConfigPath is the path to the configuration file for the distillery
|
||||
}
|
||||
|
||||
// ParamsFromEnv creates a new set of parameters from the environment.
|
||||
// Uses [ReadBaseDirectory] or falls back to [BaseDirectoryDefault].
|
||||
func ParamsFromEnv() (params Params, err error) {
|
||||
|
||||
// try to read the base directory!
|
||||
value, err := ReadBaseDirectory()
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
params.ConfigPath = BaseDirectoryDefault
|
||||
case err == nil:
|
||||
params.ConfigPath = value
|
||||
default:
|
||||
return params, err
|
||||
}
|
||||
|
||||
// and add the configuration file name to it!
|
||||
params.ConfigPath = filepath.Join(params.ConfigPath, ConfigFile)
|
||||
return params, nil
|
||||
}
|
||||
26
internal/core/requirements.go
Normal file
26
internal/core/requirements.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"github.com/tkw1536/goprogram"
|
||||
"github.com/tkw1536/goprogram/meta"
|
||||
)
|
||||
|
||||
// Requirements are requirements for the WissKI Distillery
|
||||
type Requirements struct {
|
||||
// Do we need an installed distillery?
|
||||
NeedsDistillery bool
|
||||
}
|
||||
|
||||
// AllowsFlag checks if the provided flag may be passed to fullfill this requirement
|
||||
// By default it is used only for help page generation, and may be inaccurate.
|
||||
func (r Requirements) AllowsFlag(flag meta.Flag) bool {
|
||||
return r.NeedsDistillery
|
||||
}
|
||||
|
||||
// Validate validates if this requirement is fullfilled for the provided global flags.
|
||||
// It should return either nil, or an error of type exit.Error.
|
||||
//
|
||||
// Validate does not take into account AllowsOption, see ValidateAllowedOptions.
|
||||
func (r Requirements) Validate(arguments goprogram.Arguments[Flags]) error {
|
||||
return nil
|
||||
}
|
||||
9
internal/core/runtime.go
Normal file
9
internal/core/runtime.go
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"embed"
|
||||
)
|
||||
|
||||
// Runtime contains runtime resources to be installed into any instance
|
||||
//go:embed all:runtime
|
||||
var Runtime embed.FS
|
||||
2
internal/core/runtime/README
Normal file
2
internal/core/runtime/README
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
Files in this folder are utility scripts to be used from within individual WissKI instances.
|
||||
They are mounted under runtime/ and should be used with care.
|
||||
16
internal/core/runtime/blind_update.sh
Normal file
16
internal/core/runtime/blind_update.sh
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This utility script can be used to blindly update all dependencies to their latest versions.
|
||||
# It does not perform any checking whatsoever.
|
||||
|
||||
# update the main modules
|
||||
cd /var/www/data/project || exit 1
|
||||
chmod u+rw web/sites/default/
|
||||
composer update
|
||||
|
||||
# update the db
|
||||
drush -y updatedb
|
||||
|
||||
# update the wisski dependencies
|
||||
cd /var/www/data/project/web/modules/contrib/wisski || exit 1
|
||||
composer update
|
||||
25
internal/core/runtime/create_admin.sh
Normal file
25
internal/core/runtime/create_admin.sh
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# read user
|
||||
USER=$1
|
||||
if [ -z "$USER" ]; then
|
||||
echo "Usage: create_admin.sh USERNAME"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# read password
|
||||
echo "Enter Password for $USER:"
|
||||
read -s PASS
|
||||
echo "Enter the same password again:"
|
||||
read -s PASS2
|
||||
|
||||
if [ "$PASS" != "$PASS2" ]; then
|
||||
echo "Passwords not equal"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
# create the user and add the admin role
|
||||
cd /var/www/data/project/
|
||||
drush user:create "$USER" --password="$PASS"
|
||||
drush user-add-role administrator "$USER"
|
||||
8
internal/core/runtime/cron.sh
Executable file
8
internal/core/runtime/cron.sh
Executable file
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This utility script can be used to run all cron tasks.
|
||||
|
||||
cd /var/www/data/project || exit 1
|
||||
export PATH=/var/www/data/project/vendor/bin:$PATH
|
||||
|
||||
drush core-cron
|
||||
22
internal/core/runtime/install_colorbox.sh
Normal file
22
internal/core/runtime/install_colorbox.sh
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# make a temporary directory and cd into it
|
||||
TEMPDIR=$(mktemp -d)
|
||||
pushd "$TEMPDIR"
|
||||
|
||||
# curl the colorbox zip and unpack it
|
||||
curl -L https://github.com/jackmoore/colorbox/archive/master.zip --output master.zip
|
||||
unzip master.zip
|
||||
|
||||
# make the directory for libraries, and remove the old colorbox installation
|
||||
chmod u+rw /var/www/data/project/web/sites/default/
|
||||
mkdir -p /var/www/data/project/web/sites/default/libraries/
|
||||
rm -rf /var/www/data/project/web/sites/default/libraries/colorbox
|
||||
|
||||
# copy over the new installation
|
||||
mv colorbox-master/ /var/www/data/project/web/sites/default/libraries/colorbox
|
||||
|
||||
# cleanup
|
||||
popd
|
||||
rm -rf "$TEMPDIR"
|
||||
6
internal/core/runtime/patch_easyrdf.sh
Executable file
6
internal/core/runtime/patch_easyrdf.sh
Executable file
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/sh
|
||||
|
||||
# This script can be used to repatch EasyRDF when needed.
|
||||
cd /var/www/data/project/web/modules/contrib/wisski || exit 1
|
||||
EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
|
||||
patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"
|
||||
6
internal/core/runtime/patch_triples.sh
Normal file
6
internal/core/runtime/patch_triples.sh
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/sh
|
||||
|
||||
# This script can be used to repatch EasyRDF when needed.
|
||||
cd /var/www/data/project/web/modules/contrib/wisski/ || exit 1
|
||||
TRIPLESTABCONTROLLER="./wisski_adapter_sparql11_pb/src/Controller/Sparql11TriplesTabController.php"
|
||||
patch -N "$TRIPLESTABCONTROLLER" < "/patch/triples.patch"
|
||||
22
internal/core/runtime/use_wisski.sh
Normal file
22
internal/core/runtime/use_wisski.sh
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# read user
|
||||
VERSION=$1
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "Usage: use_wisski.sh VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# update the main modules
|
||||
cd /var/www/data/project
|
||||
chmod u+rw web/sites/default/
|
||||
composer require "drupal/wisski:$VERSION"
|
||||
|
||||
# update the wisski dependencies
|
||||
pushd /var/www/data/project/web/modules/contrib/wisski
|
||||
composer update
|
||||
popd
|
||||
|
||||
# update the db
|
||||
drush -y updatedb
|
||||
26
internal/core/runtime/wisski_2x_3x.sh
Normal file
26
internal/core/runtime/wisski_2x_3x.sh
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# temporarily extend permissions
|
||||
chmod 777 web/sites/default
|
||||
chmod 666 web/sites/default/*settings.php
|
||||
chmod 666 web/sites/default/*services.yml
|
||||
|
||||
# update the core itself
|
||||
composer require 'drupal/internal/core-recommended:^9' 'drupal/internal/core-composer-scaffold:^9' 'drupal/internal/core-project-message:^9' --update-with-dependencies --no-update
|
||||
composer update
|
||||
composer require 'drupal/wisski'
|
||||
|
||||
# update requirements for wisski!
|
||||
pushd web/modules/contrib/wisski || exit 1
|
||||
composer update
|
||||
popd || exit 1
|
||||
|
||||
# run the update and clear the cache!
|
||||
drush updatedb --yes
|
||||
# drush cc
|
||||
|
||||
# and reset everything back to normal
|
||||
chmod 755 web/sites/default
|
||||
chmod 644 web/sites/default/*settings.php
|
||||
chmod 644 web/sites/default/*services.yml
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
// Package internal contains various utility functions.
|
||||
// These do not directly involve the distillery.
|
||||
package internal
|
||||
233
internal/env/backup.go
vendored
Normal file
233
internal/env/backup.go
vendored
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// backupDescription is a description for a backup
|
||||
type BackupDescription struct {
|
||||
Dest string // destination path
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Backup struct {
|
||||
Description BackupDescription
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{}
|
||||
|
||||
SQLErr error
|
||||
TSErr error
|
||||
|
||||
ConfigFileErr error
|
||||
ConfigFilesManifest map[string]error
|
||||
|
||||
InstanceListErr error
|
||||
InstancesManifest []Snapshot
|
||||
}
|
||||
|
||||
func (dis *Distillery) Backup(io stream.IOStream, description BackupDescription) (backup Backup) {
|
||||
backup.Description = description
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
backup.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
backup.run(io, dis)
|
||||
return
|
||||
}
|
||||
|
||||
var errBackupSkipFile = errors.New("<file not found>")
|
||||
|
||||
func (backup *Backup) run(io stream.IOStream, dis *Distillery) {
|
||||
// create a wait group, and message channel
|
||||
wg := &sync.WaitGroup{}
|
||||
messages := make(chan string, 4)
|
||||
|
||||
// backup the sql
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
sqlPath := filepath.Join(backup.Description.Dest, "sql.sql")
|
||||
messages <- sqlPath
|
||||
|
||||
sql, err := os.Create(sqlPath)
|
||||
if err != nil {
|
||||
backup.SQLErr = err
|
||||
return
|
||||
}
|
||||
defer sql.Close()
|
||||
|
||||
// directly store the result
|
||||
backup.SQLErr = dis.SQL().BackupAll(io, sql)
|
||||
}()
|
||||
|
||||
// backup the triplestore
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
tsPath := filepath.Join(backup.Description.Dest, "triplestore")
|
||||
messages <- tsPath
|
||||
|
||||
// directly store the result
|
||||
backup.TSErr = dis.Triplestore().BackupAll(tsPath)
|
||||
}()
|
||||
|
||||
// backup configuration files
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
cfgBackupDir := filepath.Join(backup.Description.Dest, "config")
|
||||
if err := os.Mkdir(cfgBackupDir, fs.ModeDir); err != nil {
|
||||
backup.ConfigFileErr = err
|
||||
return
|
||||
}
|
||||
|
||||
files := []string{
|
||||
filepath.Join(dis.Config.DeployRoot, ".env"), // TODO: put the name of the configuration file somewhere
|
||||
filepath.Join(dis.Config.DeployRoot, "wdcli"), // TODO: constant the name of the executable
|
||||
dis.Config.SelfOverridesFile,
|
||||
dis.Config.GlobalAuthorizedKeysFile,
|
||||
}
|
||||
|
||||
backup.ConfigFilesManifest = make(map[string]error, len(files))
|
||||
for _, src := range files {
|
||||
if !fsx.IsFile(src) {
|
||||
backup.ConfigFilesManifest[src] = errBackupSkipFile
|
||||
continue
|
||||
}
|
||||
dest := filepath.Join(cfgBackupDir, filepath.Base(src))
|
||||
|
||||
// copy the config file and store the error message
|
||||
messages <- src
|
||||
backup.ConfigFilesManifest[src] = fsx.CopyFile(dest, src)
|
||||
}
|
||||
}()
|
||||
|
||||
// backup instances
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
instancesBackupDir := filepath.Join(backup.Description.Dest, "instances")
|
||||
if err := os.Mkdir(instancesBackupDir, fs.ModeDir); err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
// list all instances
|
||||
instances, err := dis.AllInstances()
|
||||
if err != nil {
|
||||
backup.InstanceListErr = err
|
||||
return
|
||||
}
|
||||
|
||||
iochild := stream.NewIOStream(io.Stderr, io.Stderr, nil, 0)
|
||||
|
||||
backup.InstancesManifest = make([]Snapshot, len(instances))
|
||||
for i, instance := range instances {
|
||||
backup.InstancesManifest[i] = func() Snapshot {
|
||||
dir := filepath.Join(instancesBackupDir, instance.Slug)
|
||||
if err := os.Mkdir(dir, fs.ModeDir); err != nil {
|
||||
return Snapshot{
|
||||
ErrPanic: err,
|
||||
}
|
||||
}
|
||||
|
||||
messages <- dir
|
||||
return instance.Snapshot(iochild, SnapshotDescription{
|
||||
Dest: dir,
|
||||
})
|
||||
}()
|
||||
}
|
||||
|
||||
}()
|
||||
|
||||
// wait for the group, then close the message channel.
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(messages)
|
||||
}()
|
||||
|
||||
// print out all the messages
|
||||
for message := range messages {
|
||||
io.Println(message)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this backup.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (backup Backup) WriteReport(io stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(backup.Description.Dest, "report.txt")
|
||||
io.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := os.Create(reportPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = fmt.Fprintf(report, "%#v\n", backup)
|
||||
return err
|
||||
}, io, "Writing backup report")
|
||||
}
|
||||
|
||||
// ShouldPrune determines if a file with the provided modtime
|
||||
func (dis *Distillery) ShouldPrune(modtime time.Time) bool {
|
||||
return time.Since(modtime) > time.Duration(dis.Config.MaxBackupAge)*24*time.Hour
|
||||
}
|
||||
|
||||
// PruneBackups prunes all backups older than the maximum backup age
|
||||
func (dis *Distillery) PruneBackups(io stream.IOStream) error {
|
||||
sPath := dis.SnapshotsArchivePath()
|
||||
|
||||
// list all the files
|
||||
entries, err := os.ReadDir(sPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
// skip directories
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
// grab info about the file
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check if it should be pruned!
|
||||
if !dis.ShouldPrune(info.ModTime()) {
|
||||
continue
|
||||
}
|
||||
|
||||
// assemble path, and then remove the file!
|
||||
path := filepath.Join(sPath, entry.Name())
|
||||
io.Printf("Removing %s cause it is older than %d days", path, dis.Config.MaxBackupAge)
|
||||
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
126
internal/env/component.go
vendored
Normal file
126
internal/env/component.go
vendored
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/resolver"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/self"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component/web"
|
||||
)
|
||||
|
||||
// components holds the various components of the distillery
|
||||
// It is inlined into the [Distillery] struct, and initialized using [makeComponent].
|
||||
type components struct {
|
||||
// m protects the fields below
|
||||
m sync.Mutex
|
||||
|
||||
// each component is only initialized once
|
||||
web *web.Web
|
||||
self *self.Self
|
||||
resolver *resolver.Resolver
|
||||
dis *dis.Dis
|
||||
ssh *ssh.SSH
|
||||
ts *triplestore.Triplestore
|
||||
sql *sql.SQL
|
||||
}
|
||||
|
||||
// makeComponent makes or returns a component inside the [component] struct of the distillery
|
||||
//
|
||||
// C is the type of component to initialize. It must be backed by a pointer, or makeComponent will panic.
|
||||
//
|
||||
// dis is the distillery to initialize components for
|
||||
// field is a pointer to the appropriate struct field within the distillery components
|
||||
// init is called with a new non-nil component to initialize it. It may be nil, to indicate no initialization is required.
|
||||
//
|
||||
// makeComponent returns the new or existing component instance
|
||||
func makeComponent[C component.Component](dis *Distillery, field *C, init func(C)) C {
|
||||
dis.components.m.Lock()
|
||||
defer dis.components.m.Unlock()
|
||||
|
||||
// get the typeof C and make sure that it is a pointer type!
|
||||
typC := reflect.TypeOf((*C)(nil)).Elem()
|
||||
if typC.Kind() != reflect.Pointer {
|
||||
panic("makeComponent: C must be backed by a pointer")
|
||||
}
|
||||
|
||||
// if the component is non-nil, then it has already been initialized
|
||||
if !reflect.ValueOf(*field).IsNil() {
|
||||
return *field
|
||||
}
|
||||
|
||||
// create a new element, and call the initializer (if requested)
|
||||
*field = reflect.New(typC.Elem()).Interface().(C)
|
||||
if init != nil {
|
||||
init(*field)
|
||||
}
|
||||
|
||||
// apply the base configuration
|
||||
base := (*field).Base()
|
||||
base.Config = dis.Config
|
||||
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", (*field).Name())
|
||||
|
||||
// and eventually return it
|
||||
return *field
|
||||
}
|
||||
|
||||
// Components returns all components of the distillery
|
||||
func (dis *Distillery) Components() []component.Component {
|
||||
return []component.Component{
|
||||
dis.Web(),
|
||||
dis.Self(),
|
||||
dis.Resolver(),
|
||||
dis.Dis(),
|
||||
dis.SSH(),
|
||||
dis.Triplestore(),
|
||||
dis.SQL(),
|
||||
}
|
||||
}
|
||||
|
||||
func (dis *Distillery) Web() *web.Web {
|
||||
return makeComponent(dis, &dis.components.web, nil)
|
||||
}
|
||||
|
||||
func (dis *Distillery) Self() *self.Self {
|
||||
return makeComponent(dis, &dis.components.self, nil)
|
||||
}
|
||||
|
||||
func (dis *Distillery) Resolver() *resolver.Resolver {
|
||||
return makeComponent(dis, &dis.components.resolver, func(resolver *resolver.Resolver) {
|
||||
resolver.ConfigName = "prefix.cfg" // TODO: Move into core?
|
||||
resolver.Executable = dis.CurrentExecutable()
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Distillery) Dis() *dis.Dis {
|
||||
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
|
||||
ddis.Executable = d.CurrentExecutable()
|
||||
})
|
||||
}
|
||||
|
||||
func (dis *Distillery) SSH() *ssh.SSH {
|
||||
return makeComponent(dis, &dis.components.ssh, nil)
|
||||
}
|
||||
|
||||
func (dis *Distillery) SQL() *sql.SQL {
|
||||
return makeComponent(dis, &dis.components.sql, func(sql *sql.SQL) {
|
||||
sql.ServerURL = dis.Upstream.SQL
|
||||
sql.PollContext = dis.Context()
|
||||
sql.PollInterval = time.Second
|
||||
})
|
||||
}
|
||||
|
||||
func (dis *Distillery) Triplestore() *triplestore.Triplestore {
|
||||
return makeComponent(dis, &dis.components.ts, func(ts *triplestore.Triplestore) {
|
||||
ts.BaseURL = "http://" + dis.Upstream.Triplestore
|
||||
ts.PollContext = dis.Context()
|
||||
ts.PollInterval = time.Second
|
||||
})
|
||||
}
|
||||
63
internal/env/distillery.go
vendored
Normal file
63
internal/env/distillery.go
vendored
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
)
|
||||
|
||||
// Distillery represents an interface to the running distillery.
|
||||
type Distillery struct {
|
||||
// Config holds the configuration of the distillery.
|
||||
// It is read directly from a configuration file.
|
||||
Config *config.Config
|
||||
|
||||
// Upstream holds information to connect to the various running
|
||||
// distillery components.
|
||||
//
|
||||
// NOTE(twiesing): This is intended to eventually allow full remote management of the distillery.
|
||||
// But for now this will just hold upstream configuration.
|
||||
Upstream Upstream
|
||||
|
||||
// components hold references to the various components of the distillery.
|
||||
components
|
||||
}
|
||||
|
||||
// Upstream are the upstream urls connecting to the various external components.
|
||||
type Upstream struct {
|
||||
SQL string
|
||||
Triplestore string
|
||||
}
|
||||
|
||||
// Context returns a new Context belonging to this distillery
|
||||
func (dis *Distillery) Context() context.Context {
|
||||
return context.Background()
|
||||
}
|
||||
|
||||
// ExecutablePath returns the path to the executable of this distillery.
|
||||
func (dis *Distillery) ExecutablePath() string {
|
||||
return filepath.Join(dis.Config.DeployRoot, core.Executable)
|
||||
}
|
||||
|
||||
// UsingDistilleryExecutable checks if the current process
|
||||
func (dis *Distillery) UsingDistilleryExecutable() bool {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return fsx.SameFile(exe, dis.ExecutablePath())
|
||||
}
|
||||
|
||||
// CurrentExecutable returns the path to the current executable being used.
|
||||
// When it does not exist, falls back to the default executable.
|
||||
func (dis *Distillery) CurrentExecutable() string {
|
||||
exe, err := os.Executable()
|
||||
if err != nil || !fsx.IsFile(exe) {
|
||||
return dis.ExecutablePath()
|
||||
}
|
||||
return exe
|
||||
}
|
||||
62
internal/env/init.go
vendored
Normal file
62
internal/env/init.go
vendored
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/core"
|
||||
"github.com/tkw1536/goprogram/exit"
|
||||
)
|
||||
|
||||
var errNoConfigFile = exit.Error{
|
||||
ExitCode: exit.ExitGeneralArguments,
|
||||
Message: "Configuration File does not exist",
|
||||
}
|
||||
|
||||
var errOpenConfig = exit.Error{
|
||||
ExitCode: exit.ExitGeneralArguments,
|
||||
Message: "error loading configuration file: %s",
|
||||
}
|
||||
|
||||
// NewDistillery creates a new distillery object from a set of parameters and requirements
|
||||
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (env *Distillery, err error) {
|
||||
env = &Distillery{
|
||||
Upstream: Upstream{
|
||||
SQL: "127.0.0.1:3306",
|
||||
Triplestore: "127.0.0.1:7200",
|
||||
},
|
||||
}
|
||||
|
||||
if flags.InternalInDocker {
|
||||
env.Upstream.SQL = "sql:3306"
|
||||
env.Upstream.Triplestore = "triplestore:7200"
|
||||
}
|
||||
|
||||
// if we don't need to load the config, there is nothing to do
|
||||
if !req.NeedsDistillery {
|
||||
return
|
||||
}
|
||||
|
||||
// try to find the configuration file
|
||||
cfg := flags.ConfigPath // command line flags first
|
||||
if cfg == "" {
|
||||
cfg = params.ConfigPath // then globally provided files
|
||||
}
|
||||
if cfg == "" {
|
||||
return nil, errNoConfigFile
|
||||
}
|
||||
|
||||
// open the config file!
|
||||
f, err := os.Open(params.ConfigPath)
|
||||
if err != nil {
|
||||
return nil, errOpenConfig.WithMessageF(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// unmarshal the config
|
||||
env.Config = &config.Config{
|
||||
ConfigPath: cfg,
|
||||
}
|
||||
err = env.Config.Unmarshal(f)
|
||||
return
|
||||
}
|
||||
408
internal/env/instances.go
vendored
Normal file
408
internal/env/instances.go
vendored
Normal file
|
|
@ -0,0 +1,408 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/alessio/shellescape"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/exit"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
var errNoBookkeeping = exit.Error{
|
||||
Message: "instance %q does not exist in bookkeeping table",
|
||||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
var ErrInstanceNotFound = exit.Error{
|
||||
Message: "instance not found",
|
||||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
var errSQL = exit.Error{
|
||||
Message: "Unknown SQL Error %s",
|
||||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
// Instance returns the instance of the WissKI Distillery with the provided slug
|
||||
func (dis *Distillery) Instance(slug string) (i Instance, err error) {
|
||||
sql := dis.SQL()
|
||||
if err := sql.Wait(); err != nil {
|
||||
return i, err
|
||||
}
|
||||
|
||||
table, err := sql.OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return i, err
|
||||
}
|
||||
|
||||
// find the instance by slug
|
||||
query := table.Where(&bookkeeping.Instance{Slug: slug}).Find(&i.Instance)
|
||||
switch {
|
||||
case query.Error != nil:
|
||||
return i, errSQL.WithMessageF(query.Error)
|
||||
case query.RowsAffected == 0:
|
||||
return i, ErrInstanceNotFound
|
||||
default:
|
||||
i.dis = dis
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
|
||||
// HasInstance checks if the provided instance exists in the bookeeping table
|
||||
func (dis *Distillery) HasInstance(slug string) (ok bool, err error) {
|
||||
sql := dis.SQL()
|
||||
if err := sql.Wait(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
table, err := sql.OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
query := table.Select("count(*) > 0").Where("slug = ?", slug).Find(&ok)
|
||||
if query.Error != nil {
|
||||
return false, errSQL.WithMessageF(query.Error)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Instances is like InstancesWith, except that when no slugs are provided, it calls AllInstances.
|
||||
func (dis *Distillery) Instances(slugs ...string) ([]Instance, error) {
|
||||
if len(slugs) == 0 {
|
||||
return dis.AllInstances()
|
||||
}
|
||||
return dis.InstancesWith(slugs...)
|
||||
}
|
||||
|
||||
// AllInstances returns all instances of the WissKI Distillery in consistent order.
|
||||
//
|
||||
// There is no guarantee that this order remains identical between different api releases; however subsequent invocations are guaranteed to return the same order.
|
||||
func (dis *Distillery) AllInstances() ([]Instance, error) {
|
||||
return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
|
||||
return table
|
||||
})
|
||||
}
|
||||
|
||||
// InstancesWith returns all instances where the slug is in the provided list of names.
|
||||
// The returned instances are reordered in a consistent order.
|
||||
func (dis *Distillery) InstancesWith(slugs ...string) ([]Instance, error) {
|
||||
return dis.findInstances(true, func(table *gorm.DB) *gorm.DB {
|
||||
return table.Where("slug IN ?", slugs)
|
||||
})
|
||||
}
|
||||
|
||||
// findInstances finds instance objects based on a query in the bookkeeping table
|
||||
func (dis *Distillery) findInstances(order bool, query func(table *gorm.DB) *gorm.DB) (instances []Instance, err error) {
|
||||
sql := dis.SQL()
|
||||
if err := sql.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open the bookkeeping table
|
||||
table, err := sql.OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// prepare a query
|
||||
find := table
|
||||
if order {
|
||||
find = find.Order(clause.OrderByColumn{Column: clause.Column{Name: "slug"}, Desc: false})
|
||||
}
|
||||
if query != nil {
|
||||
find = query(find)
|
||||
}
|
||||
|
||||
// fetch bookkeeping instances
|
||||
var bks []bookkeeping.Instance
|
||||
find = find.Find(&bks)
|
||||
if find.Error != nil {
|
||||
return nil, errSQL.WithMessageF(find.Error)
|
||||
}
|
||||
|
||||
// make proper instances
|
||||
instances = make([]Instance, len(bks))
|
||||
for i, bk := range bks {
|
||||
instances[i].Instance = bk
|
||||
instances[i].dis = dis
|
||||
}
|
||||
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
// Instance represents a bookkeeping instance
|
||||
type Instance struct {
|
||||
bookkeeping.Instance
|
||||
|
||||
// Credentials for the drupal instance
|
||||
DrupalUsername string
|
||||
DrupalPassword string
|
||||
|
||||
dis *Distillery
|
||||
}
|
||||
|
||||
// Update updates the bookkeeping table with this instance.
|
||||
func (instance *Instance) Update() error {
|
||||
db, err := instance.dis.SQL().OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// it has never been created => we need to create it in the database
|
||||
if instance.Instance.Created.IsZero() {
|
||||
return db.Create(&instance.Instance).Error
|
||||
}
|
||||
|
||||
// Update based on the primary key!
|
||||
return db.Where("pk = ?", instance.Instance.Pk).Updates(&instance.Instance).Error
|
||||
}
|
||||
|
||||
// Delete deletes this instance from the bookkeeping table
|
||||
func (instance *Instance) Delete() error {
|
||||
db, err := instance.dis.SQL().OpenBookkeeping(false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// doesn't exist => nothing to delete
|
||||
if instance.Instance.Created.IsZero() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// delete it directly
|
||||
return db.Delete(&instance.Instance).Error
|
||||
}
|
||||
|
||||
// Shell executes a shell command inside the
|
||||
func (instance Instance) Shell(io stream.IOStream, argv ...string) (int, error) {
|
||||
return instance.Stack().Exec(io, "barrel", "/user_shell.sh", argv...)
|
||||
}
|
||||
|
||||
// Domain returns the full domain name of this instance
|
||||
func (instance Instance) Domain() string {
|
||||
return fmt.Sprintf("%s.%s", instance.Slug, instance.dis.Config.DefaultDomain)
|
||||
}
|
||||
|
||||
// URL returns the public URL of this instance
|
||||
func (instance Instance) URL() *url.URL {
|
||||
// setup domain and path
|
||||
url := &url.URL{
|
||||
Host: instance.Domain(),
|
||||
Path: "/",
|
||||
}
|
||||
|
||||
// use http or https scheme depending on if the distillery has it enabled
|
||||
if instance.dis.Config.HTTPSEnabled() {
|
||||
url.Scheme = "https"
|
||||
} else {
|
||||
url.Scheme = "http"
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
//go:embed all:instances/barrel instances/barrel.env
|
||||
var barrelResources embed.FS
|
||||
|
||||
// Stack represents a stack representing this instance
|
||||
func (instance Instance) Stack() component.Installable {
|
||||
return component.Installable{
|
||||
Stack: component.Stack{
|
||||
Dir: instance.FilesystemBase,
|
||||
},
|
||||
|
||||
Resources: barrelResources,
|
||||
ContextPath: filepath.Join("instances", "barrel"),
|
||||
EnvPath: filepath.Join("instances", "barrel.env"),
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"DATA_PATH": filepath.Join(instance.FilesystemBase, "data"),
|
||||
|
||||
"SLUG": instance.Slug,
|
||||
"VIRTUAL_HOST": instance.Domain(),
|
||||
|
||||
"LETSENCRYPT_HOST": instance.dis.Config.IfHttps(instance.Domain()),
|
||||
"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
|
||||
|
||||
"RUNTIME_DIR": instance.dis.RuntimeDir(),
|
||||
"GLOBAL_AUTHORIZED_KEYS_FILE": instance.dis.Config.GlobalAuthorizedKeysFile,
|
||||
},
|
||||
|
||||
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
|
||||
MakeDirs: []string{"data", ".composer"},
|
||||
|
||||
TouchFiles: []string{
|
||||
filepath.Join("data", "authorized_keys"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
//go:embed all:instances/reserve instances/reserve.env
|
||||
var reserveResources embed.FS
|
||||
|
||||
func (instance Instance) ReserveStack() component.Installable {
|
||||
return component.Installable{
|
||||
Stack: component.Stack{
|
||||
Dir: instance.FilesystemBase,
|
||||
},
|
||||
|
||||
Resources: reserveResources,
|
||||
ContextPath: filepath.Join("instances", "reserve"),
|
||||
EnvPath: filepath.Join("instances", "reserve.env"),
|
||||
|
||||
EnvContext: map[string]string{
|
||||
"VIRTUAL_HOST": instance.Domain(),
|
||||
|
||||
"LETSENCRYPT_HOST": instance.dis.Config.IfHttps(instance.Domain()),
|
||||
"LETSENCRYPT_EMAIL": instance.dis.Config.IfHttps(instance.dis.Config.CertbotEmail),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Provision provisions an instance, assuming that the required databases already exist.
|
||||
func (instance Instance) Provision(io stream.IOStream) error {
|
||||
|
||||
// create the basic st!
|
||||
st := instance.Stack()
|
||||
if err := st.Install(io, component.InstallationContext{}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pull and build the stack!
|
||||
if err := st.Update(io, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
provisionParams := []string{
|
||||
instance.Domain(),
|
||||
|
||||
instance.SqlDatabase,
|
||||
instance.SqlUser,
|
||||
instance.SqlPassword,
|
||||
|
||||
instance.GraphDBRepository,
|
||||
instance.GraphDBUser,
|
||||
instance.GraphDBPassword,
|
||||
|
||||
instance.DrupalUsername,
|
||||
instance.DrupalPassword,
|
||||
|
||||
"", // TODO: DrupalVersion
|
||||
"", // TODO: WissKIVersion
|
||||
}
|
||||
|
||||
// escape the parameter
|
||||
for i, param := range provisionParams {
|
||||
provisionParams[i] = shellescape.Quote(param)
|
||||
}
|
||||
|
||||
// figure out the provision script
|
||||
// TODO: Move the provision script into the control plane!
|
||||
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
|
||||
|
||||
code, err := st.Run(io, true, "barrel", "/bin/bash", "-c", provisionScript)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 0 {
|
||||
return errors.New("Unable to run provision script")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (instance *Instance) NoPrefix() bool {
|
||||
return fsx.IsFile(filepath.Join(instance.FilesystemBase, "prefixes.skip"))
|
||||
}
|
||||
|
||||
var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
|
||||
|
||||
// PrefixConfig returns the prefix config belonging to this instance.
|
||||
func (instance *Instance) PrefixConfig() (config string, err error) {
|
||||
// if the user requested to skip the prefix, then don't do anything with it!
|
||||
if instance.NoPrefix() {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
var builder strings.Builder
|
||||
|
||||
// domain
|
||||
builder.WriteString(instance.URL().String() + ":")
|
||||
builder.WriteString("\n")
|
||||
|
||||
// default prefixes
|
||||
wu := stream.NewIOStream(&builder, nil, nil, 0)
|
||||
code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
|
||||
if err != nil || code != 0 {
|
||||
return "", errPrefixExecFailed
|
||||
}
|
||||
|
||||
// custom prefixes
|
||||
prefixPath := filepath.Join(instance.FilesystemBase, "prefixes")
|
||||
if fsx.IsFile(prefixPath) {
|
||||
prefix, err := os.Open(prefixPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer prefix.Close()
|
||||
if _, err := io.Copy(&builder, prefix); err != nil {
|
||||
return "", err
|
||||
}
|
||||
builder.WriteString("\n")
|
||||
}
|
||||
|
||||
// and done!
|
||||
return builder.String(), nil
|
||||
}
|
||||
|
||||
var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder")
|
||||
|
||||
// ExportPathbuilders writes pathbuilders into the directory dest
|
||||
func (instance *Instance) ExportPathbuilders(dest string) error {
|
||||
// export all the pathbuilders into the buffer
|
||||
var buffer bytes.Buffer
|
||||
wu := stream.NewIOStream(&buffer, nil, nil, 0)
|
||||
code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php")
|
||||
if err != nil || code != 0 {
|
||||
return errPathbuildersExecFailed
|
||||
}
|
||||
|
||||
// decode them as a json array
|
||||
var pathbuilders map[string]string
|
||||
if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// sort the names of the pathbuilders
|
||||
names := maps.Keys(pathbuilders)
|
||||
slices.Sort(names)
|
||||
|
||||
// write each into a file!
|
||||
for _, name := range names {
|
||||
pbxml := []byte(pathbuilders[name])
|
||||
name := filepath.Join(dest, fmt.Sprintf("%s.xml", name))
|
||||
if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
10
internal/env/instances/barrel.env
vendored
Normal file
10
internal/env/instances/barrel.env
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
DATA_PATH=${DATA_PATH}
|
||||
RUNTIME_DIR=${RUNTIME_DIR}
|
||||
|
||||
SLUG=${SLUG}
|
||||
VIRTUAL_HOST=${VIRTUAL_HOST}
|
||||
|
||||
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
|
||||
|
||||
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
|
||||
8
internal/env/instances/barrel/.dockerignore
vendored
Normal file
8
internal/env/instances/barrel/.dockerignore
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# Ignore everything
|
||||
*
|
||||
|
||||
# allow the following files:
|
||||
!conf/*
|
||||
!scripts/*
|
||||
!patch/*
|
||||
!wisskiutils/*
|
||||
28
internal/env/instances/barrel/.env.sample
vendored
Normal file
28
internal/env/instances/barrel/.env.sample
vendored
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#######################
|
||||
# Meta Settings
|
||||
#######################
|
||||
|
||||
# Real path for volumes to be stored
|
||||
DATA_PATH=/var/www/deploy/instances/example.slug/data
|
||||
UTILS_DIR=/var/www/deploy/runtime/utils/
|
||||
|
||||
#######################
|
||||
### Web Server settings
|
||||
#######################
|
||||
# the hostname for the website
|
||||
VIRTUAL_HOST=example.com
|
||||
|
||||
# optional letsencrypt support
|
||||
# when blank, ignore
|
||||
LETSENCRYPT_HOST=
|
||||
LETSENCRYPT_EMAIL=
|
||||
|
||||
### SQL settings
|
||||
MYSQL_HOST=mysql
|
||||
MYSQL_USER=user
|
||||
MYSQL_PASS=pass
|
||||
|
||||
### GraphDB settings
|
||||
GRAPHDB_HOST=graphdb
|
||||
GRAPHDB_USER=user
|
||||
GRAPHDB_PASS=pass
|
||||
106
internal/env/instances/barrel/Dockerfile
vendored
Normal file
106
internal/env/instances/barrel/Dockerfile
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
FROM docker.io/library/php:8.0-apache-bullseye
|
||||
ARG COMPOSER_VERSION=2.3.8
|
||||
WORKDIR /var/www
|
||||
|
||||
# install and enable the various required php extension
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
default-mysql-client \
|
||||
git \
|
||||
imagemagick \
|
||||
libcurl4-openssl-dev \
|
||||
libfreetype6-dev \
|
||||
libicu-dev \
|
||||
libjpeg62-turbo-dev \
|
||||
libpng-dev \
|
||||
libssh2-1-dev \
|
||||
libwebp-dev \
|
||||
libxml2-dev \
|
||||
libxpm-dev \
|
||||
sudo \
|
||||
unzip \
|
||||
vim \
|
||||
zip \
|
||||
&& \
|
||||
docker-php-source extract && \
|
||||
mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" && \
|
||||
pear config-set php_ini "$PHP_INI_DIR/php.ini" && \
|
||||
docker-php-ext-configure gd \
|
||||
--enable-gd \
|
||||
--with-webp \
|
||||
--with-jpeg \
|
||||
--with-xpm \
|
||||
--with-freetype \
|
||||
--enable-gd-jis-conv \
|
||||
&& \
|
||||
docker-php-ext-install \
|
||||
curl \
|
||||
gd \
|
||||
intl \
|
||||
mysqli \
|
||||
opcache \
|
||||
pdo_mysql \
|
||||
soap \
|
||||
xml \
|
||||
&& \
|
||||
pecl install xmlrpc-1.0.0RC3 && \
|
||||
pecl install ssh2-1.3.1 && \
|
||||
pecl install apcu-5.1.21 && \
|
||||
pecl install uploadprogress-2.0.2 && \
|
||||
docker-php-ext-enable \
|
||||
apcu \
|
||||
curl \
|
||||
gd \
|
||||
intl \
|
||||
mysqli \
|
||||
mysqli \
|
||||
opcache \
|
||||
pdo_mysql \
|
||||
soap \
|
||||
ssh2 \
|
||||
uploadprogress \
|
||||
xml \
|
||||
xmlrpc \
|
||||
&& \
|
||||
docker-php-source delete
|
||||
|
||||
# enable the apache rewrite mod
|
||||
RUN a2enmod rewrite
|
||||
|
||||
# install composer and add it to path
|
||||
RUN curl -sS https://getcomposer.org/installer | php -- --version=$COMPOSER_VERSION && \
|
||||
mv composer.phar /usr/local/bin/composer
|
||||
ENV PATH "/usr/local/bin:/var/www/data/project/vendor/bin:$PATH"
|
||||
|
||||
# remove default configuration
|
||||
RUN rm /etc/apache2/sites-available/*.conf && \
|
||||
rm /etc/apache2/sites-enabled/*.conf
|
||||
|
||||
ADD patch/easyrdf.patch /patch/easyrdf.patch
|
||||
ADD patch/triples.patch /patch/triples.patch
|
||||
|
||||
# Add wisski configuration
|
||||
ADD conf/ports.conf /etc/apache2/ports.conf
|
||||
ADD conf/wisski.conf /etc/apache2/sites-available/wisski.conf
|
||||
ADD conf/wisski.ini /usr/local/etc/php/conf.d/wisski.ini
|
||||
RUN a2ensite wisski
|
||||
|
||||
# volumes for composer
|
||||
VOLUME /var/www/.composer
|
||||
VOLUME /var/www/data
|
||||
|
||||
# Add and configure the entrypoint
|
||||
ADD scripts/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
|
||||
CMD ["apache2-foreground"]
|
||||
|
||||
# Add the provision script and WissKI utils
|
||||
ADD scripts/provision_container.sh /provision_container.sh
|
||||
ADD wisskiutils/ /wisskiutils
|
||||
|
||||
# Add the user_shell.sh
|
||||
ADD scripts/user_shell.sh /user_shell.sh
|
||||
|
||||
# expose port 8080
|
||||
EXPOSE 8080
|
||||
4
internal/env/instances/barrel/conf/ports.conf
vendored
Normal file
4
internal/env/instances/barrel/conf/ports.conf
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# This file configures where apache should listen.
|
||||
# Because we are running as a limited user, we want to listen on a high port.
|
||||
# For this we use port 8080
|
||||
Listen 8080
|
||||
24
internal/env/instances/barrel/conf/wisski.conf
vendored
Normal file
24
internal/env/instances/barrel/conf/wisski.conf
vendored
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
<VirtualHost *:8080>
|
||||
# the document root -- /var/www/data/project/web
|
||||
DocumentRoot /var/www/data/project/web
|
||||
|
||||
<Directory /var/www/data/project/web>
|
||||
# add types for .owl and .rdf
|
||||
AddType application/rdf+xml .owl
|
||||
AddType application/rdf+xml .rdf
|
||||
|
||||
# Rewrite the 'ontology' directory
|
||||
RewriteEngine On
|
||||
RewriteOptions InheritDownBefore
|
||||
ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,END]
|
||||
ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [END]
|
||||
|
||||
# Allow overrides of symlinks
|
||||
Options Indexes FollowSymLinks
|
||||
AllowOverride All
|
||||
Require all granted
|
||||
</Directory>
|
||||
|
||||
ErrorLog /dev/stderr
|
||||
CustomLog /dev/stdout combined
|
||||
</VirtualHost>
|
||||
14
internal/env/instances/barrel/conf/wisski.ini
vendored
Normal file
14
internal/env/instances/barrel/conf/wisski.ini
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
; File Uploads up to 1GB
|
||||
file_uploads = On
|
||||
upload_max_filesize = 1000M
|
||||
post_max_size = 1000M
|
||||
|
||||
; Composer uses an absurd amount of memory
|
||||
; 4GB ought to be enough
|
||||
memory_limit = 4G
|
||||
|
||||
; Increase various limits for some long running WissKI operations
|
||||
max_execution_time = 3000
|
||||
max_input_time = 600
|
||||
max_input_nesting_level = 640
|
||||
max_input_vars = 10000
|
||||
33
internal/env/instances/barrel/docker-compose.yml
vendored
Normal file
33
internal/env/instances/barrel/docker-compose.yml
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
barrel:
|
||||
build: .
|
||||
restart: always
|
||||
hostname: ${VIRTUAL_HOST}.wisski
|
||||
environment:
|
||||
# port and hostname for this image to use
|
||||
VIRTUAL_HOST: ${VIRTUAL_HOST}
|
||||
VIRTUAL_PORT: 8080
|
||||
|
||||
# optional letsencrypt email
|
||||
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
|
||||
|
||||
# label it with the current slug
|
||||
labels:
|
||||
eu.wiss-ki.barrel.slug: ${SLUG}
|
||||
eu.wiss-ki.barrel.authfile: /var/www/.ssh/authorized_keys,/var/www/.ssh/global_authorized_keys
|
||||
|
||||
# volumes that are mounted
|
||||
volumes:
|
||||
- ${GLOBAL_AUTHORIZED_KEYS_FILE}:/var/www/.ssh/global_authorized_keys:ro
|
||||
- ${DATA_PATH}/.composer:/var/www/.composer
|
||||
- ${DATA_PATH}/data:/var/www/data
|
||||
- ${DATA_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
|
||||
- ${RUNTIME_DIR}:/runtime:ro
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
4
internal/env/instances/barrel/patch/easyrdf.patch
vendored
Normal file
4
internal/env/instances/barrel/patch/easyrdf.patch
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
281c281
|
||||
< if (preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]+)|", $status, $m)) {
|
||||
---
|
||||
> if(preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]*)|", $status, $m)) {
|
||||
8
internal/env/instances/barrel/patch/triples.patch
vendored
Normal file
8
internal/env/instances/barrel/patch/triples.patch
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
100c100
|
||||
< if($result->o instanceof \EasyRdf_Resource) {
|
||||
---
|
||||
> if($result->o instanceof \EasyRdf\Resource) {
|
||||
118c118
|
||||
< $object_text = $result->o->getValue();
|
||||
---
|
||||
> $object_text = $result->o->dumpValue('string');
|
||||
11
internal/env/instances/barrel/scripts/entrypoint.sh
vendored
Executable file
11
internal/env/instances/barrel/scripts/entrypoint.sh
vendored
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script contains
|
||||
|
||||
# chown the volumes to make sure they can be read and written by the limited user
|
||||
chown www-data:www-data /var/www
|
||||
chown www-data:www-data /var/www/.composer
|
||||
chown www-data:www-data /var/www/data/
|
||||
|
||||
# run the original entrypoint
|
||||
docker-php-entrypoint "$@"
|
||||
174
internal/env/instances/barrel/scripts/provision_container.sh
vendored
Normal file
174
internal/env/instances/barrel/scripts/provision_container.sh
vendored
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
function log_info() {
|
||||
echo -e "\033[1m$1\033[0m"
|
||||
}
|
||||
|
||||
function log_ok() {
|
||||
echo -e "\033[0;32m$1\033[0m"
|
||||
}
|
||||
|
||||
log_info " => Reading configuration variables"
|
||||
|
||||
INSTANCE_DOMAIN="$1"
|
||||
echo "INSTANCE_DOMAIN=$INSTANCE_DOMAIN"
|
||||
shift 1
|
||||
|
||||
MYSQL_DATABASE="$1"
|
||||
echo "MYSQL_DATABASE=$MYSQL_DATABASE"
|
||||
MYSQL_USER="$2"
|
||||
echo "MYSQL_USER=$MYSQL_USER"
|
||||
MYSQL_PASSWORD="$3"
|
||||
echo "MYSQL_PASSWORD=$MYSQL_PASSWORD"
|
||||
|
||||
shift 3
|
||||
|
||||
GRAPHDB_REPO="$1"
|
||||
echo "GRAPHDB_REPO=$GRAPHDB_REPO"
|
||||
GRAPHDB_USER="$2"
|
||||
echo "GRAPHDB_USER=$GRAPHDB_USER"
|
||||
GRAPHDB_PASSWORD="$3"
|
||||
echo "GRAPHDB_PASSWORD=$GRAPHDB_PASSWORD"
|
||||
shift 3
|
||||
|
||||
GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
|
||||
|
||||
DRUPAL_USER="$1"
|
||||
echo "DRUPAL_USER=$DRUPAL_USER"
|
||||
DRUPAL_PASS="$2"
|
||||
echo "DRUPAL_PASS=$DRUPAL_PASS"
|
||||
shift 2
|
||||
|
||||
DRUPAL_VERSION="$1"
|
||||
echo "DRUPAL_VERSION=$DRUPAL_VERSION"
|
||||
shift 1
|
||||
|
||||
WISSKI_VERSION="$1"
|
||||
echo "WISSKI_VERSION=$WISSKI_VERSION"
|
||||
shift 1
|
||||
|
||||
log_info " => Preparing installation environment"
|
||||
BASE_DIR="/var/www/data"
|
||||
COMPOSER_DIR="$BASE_DIR/project"
|
||||
WEB_DIR="$COMPOSER_DIR/web"
|
||||
ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
|
||||
|
||||
log_info " => Creating '$COMPOSER_DIR'"
|
||||
mkdir -p "$COMPOSER_DIR"
|
||||
cd "$COMPOSER_DIR"
|
||||
|
||||
# workaround for making the drupal sites directory writable
|
||||
function drupal_sites_permission_workaround() {
|
||||
chmod -R u+w "$WEB_DIR/sites/" || true
|
||||
}
|
||||
|
||||
# install a module with composer and enable it with drush
|
||||
# Example:
|
||||
#
|
||||
# composer_install_and_enable << EOF
|
||||
# drupal/some_module:1.23 some_module
|
||||
# drupal/other_module:2.34
|
||||
# EOF
|
||||
#
|
||||
# Will install both modules, but only enable the first one.
|
||||
function composer_install_and_enable() {
|
||||
while IFS= read -r line; do
|
||||
echo "$line" | (
|
||||
read composer drush;
|
||||
drupal_sites_permission_workaround
|
||||
composer require "$composer"
|
||||
if [ -n "$drush" ]; then
|
||||
drush pm-enable --yes "$drush"
|
||||
fi
|
||||
)
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# Create a new composer project.
|
||||
log_info " => Creating composer project"
|
||||
if [ -z "${DRUPAL_VERSION}" ]; then
|
||||
composer --no-interaction create-project 'drupal/recommended-project:^9.0.0' .
|
||||
else
|
||||
composer --no-interaction create-project "drupal/recommended-project:$DRUPAL_VERSION" .
|
||||
fi
|
||||
|
||||
# needed for composer > 2.2
|
||||
composer --no-interaction config allow-plugins true
|
||||
|
||||
# Install drush so that we can automate a lot of things
|
||||
log_info " => Installing 'drush'"
|
||||
composer require drush/drush
|
||||
|
||||
# Use 'drush' to run the site-installation.
|
||||
# Here we need to use the username, password and database creds we made above.
|
||||
log_info " => Running drupal installation scripts"
|
||||
drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
|
||||
--account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
|
||||
--db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@sql/${MYSQL_DATABASE}
|
||||
drupal_sites_permission_workaround
|
||||
|
||||
# create a directory for ontologies.
|
||||
log_info " => Creating '$ONTOLOGY_DIR'"
|
||||
mkdir -p "$ONTOLOGY_DIR"
|
||||
|
||||
# Install the Wisski packages.
|
||||
log_info " => Installing Wisski packages"
|
||||
cd "$COMPOSER_DIR"
|
||||
|
||||
# install the development version when requested
|
||||
if [ -z "${WISSKI_VERSION}" ]; then
|
||||
composer require 'drupal/wisski'
|
||||
else
|
||||
composer require "drupal/wisski:$WISSKI_VERSION"
|
||||
fi
|
||||
|
||||
# Install dependencies of WissKI
|
||||
log_info " => Installing and patching Wisski dependencies"
|
||||
pushd "$WEB_DIR/modules/contrib/wisski"
|
||||
composer install
|
||||
|
||||
# Patch EasyRDF (for now)
|
||||
EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
|
||||
if [ -f "$EASYRDF_RESPONSE" ]; then
|
||||
patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"
|
||||
fi
|
||||
popd
|
||||
|
||||
log_info " => Installing and enabling additional modules"
|
||||
composer_install_and_enable << EOF
|
||||
drupal/inline_entity_form:^1.0@RC
|
||||
drupal/imagemagick
|
||||
drupal/image_effects
|
||||
drupal/colorbox
|
||||
drupal/devel:^4.1 devel
|
||||
drupal/geofield:^1.40 geofield
|
||||
drupal/geofield_map:^2.85 geofield_map
|
||||
drupal/imce:^2.4 imce
|
||||
EOF
|
||||
|
||||
log_info " => Enable Wisski modules"
|
||||
drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
|
||||
drupal_sites_permission_workaround
|
||||
|
||||
log_info " => Setting up WissKI Salz Adapter"
|
||||
drush php:script /wisskiutils/create_adapter.php "$INSTANCE_DOMAIN" "$GRAPHDB_REPO" "$GRAPHDB_HEADER"
|
||||
|
||||
log_info " => Updating TRUSTED_HOST_PATTERNS in settings.php"
|
||||
|
||||
/bin/bash /wisskiutils/set_trusted_host.sh
|
||||
|
||||
log_info " => Running initial cron"
|
||||
drush core-cron
|
||||
|
||||
log_info " => Provisioning is now complete. "
|
||||
log_ok "Your installation details are as follows:"
|
||||
function printdetails() {
|
||||
echo "URL: http://$INSTANCE_DOMAIN"
|
||||
echo "Username: $DRUPAL_USER"
|
||||
echo "Password: $DRUPAL_PASS"
|
||||
}
|
||||
printdetails
|
||||
|
||||
exit 0
|
||||
5
internal/env/instances/barrel/scripts/user_shell.sh
vendored
Executable file
5
internal/env/instances/barrel/scripts/user_shell.sh
vendored
Executable file
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script is used to start a user shell inside the docker container.
|
||||
cd "/var/www/data/project"
|
||||
sudo -u www-data "PATH=/var/www/data/project/vendor/bin:$PATH" /bin/bash "$@"
|
||||
61
internal/env/instances/barrel/wisskiutils/create_adapter.php
vendored
Normal file
61
internal/env/instances/barrel/wisskiutils/create_adapter.php
vendored
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
<?php
|
||||
|
||||
/**
|
||||
* This script will automatically create a WissKI Salz Adapter for use within the distillery.
|
||||
* It will not update any existing adapter and is rather primitive.
|
||||
*/
|
||||
|
||||
$argc = $_SERVER['argc']-3;
|
||||
$argv = array_slice($_SERVER['argv'], 3);
|
||||
|
||||
// read parameters from the command line
|
||||
if ($argc != 3) {
|
||||
die("Usage: drush php:script create_adapter.php INSTANCE_DOMAIN GRAPHDB_REPO HEADER");
|
||||
}
|
||||
$INSTANCE_DOMAIN = $argv[0];
|
||||
$GRAPHDB_REPO = $argv[1];
|
||||
$HEADER = $argv[2];
|
||||
|
||||
//
|
||||
// PROPERTIES FOR THE ADAPTER
|
||||
//
|
||||
|
||||
$id = 'default'; // id
|
||||
$type = 'sparql11_with_pb'; // plugin
|
||||
$machine_name = 'default'; // machine-name
|
||||
$label = 'Default WissKI Distillery Adapter';
|
||||
$description = 'Adapter for ' . $INSTANCE_DOMAIN; // description
|
||||
$writable = TRUE; // writable
|
||||
$is_preferred_local_store = TRUE; // is_preferred_local_store
|
||||
$header = $HEADER; // header
|
||||
$read_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO; // read_url
|
||||
$write_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO . '/statements'; // write_url
|
||||
$is_federatable = TRUE; // is_federatable
|
||||
$default_graph_uri = 'https://' . $INSTANCE_DOMAIN . '/';
|
||||
$same_as_properties = ['http://www.w3.org/2002/07/owl#sameAs']; // same_as_properties
|
||||
$ontology_graphs = []; // ontology_graphs
|
||||
|
||||
//
|
||||
// Do the creation!
|
||||
//
|
||||
|
||||
$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
|
||||
$adapter = $storage->create([
|
||||
"id" => $id,
|
||||
"label" => $label,
|
||||
"description" => $description,
|
||||
]);
|
||||
$adapter->setEngineConfig([
|
||||
"id" => $type,
|
||||
"machine-name" => $machine_name,
|
||||
"header" => $header,
|
||||
"writeable" => $writable,
|
||||
"is_preferred_local_store" => $is_preferred_local_store,
|
||||
"read_url" => $read_url,
|
||||
"write_url" => $write_url,
|
||||
"is_federatable" => $is_federatable,
|
||||
"default_graph" => $default_graph_uri,
|
||||
"same_as_properties" => $same_as_properties,
|
||||
"ontology_graphs" => $ontology_graphs,
|
||||
]);
|
||||
$adapter->save();
|
||||
63
internal/env/instances/barrel/wisskiutils/export_pathbuilder.php
vendored
Normal file
63
internal/env/instances/barrel/wisskiutils/export_pathbuilder.php
vendored
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
<?php
|
||||
|
||||
/**
|
||||
* This script will list all the URIs that this system is aware of.
|
||||
* This works by listing all the default graph uris of all the adapters.
|
||||
*/
|
||||
|
||||
use Drupal\wisski_pathbuilder\Entity\WisskiPathEntity;
|
||||
|
||||
// load all the pathbuilders
|
||||
$pbs = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->loadMultiple();
|
||||
|
||||
// map over the pathbuilders
|
||||
$xmls = array_map(function($pb) {
|
||||
$xml = new \SimpleXMLElement("<pathbuilderinterface></pathbuilderinterface>");
|
||||
|
||||
$paths = $pb->getAllPaths();
|
||||
foreach ($paths as $key => $path) {
|
||||
$id = $path->getID();
|
||||
|
||||
$path = $pb->getPbPath($id);
|
||||
|
||||
$pathChild = $xml->addChild("path");
|
||||
$pathObject = WisskiPathEntity::load($id);
|
||||
|
||||
foreach ($path as $subkey => $value) {
|
||||
|
||||
if (in_array($subkey, ['relativepath'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($subkey == "parent") {
|
||||
$subkey = "group_id";
|
||||
}
|
||||
|
||||
$pathChild->addChild($subkey, htmlspecialchars($value));
|
||||
}
|
||||
|
||||
$pathArray = $pathChild->addChild('path_array');
|
||||
foreach ($pathObject->getPathArray() as $subkey => $value) {
|
||||
$pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value);
|
||||
}
|
||||
|
||||
$pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty()));
|
||||
$pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName()));
|
||||
$pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb()));
|
||||
$pathChild->addChild('description', htmlspecialchars($pathObject->getDescription()));
|
||||
$pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid()));
|
||||
if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") {
|
||||
$pathChild->addChild('is_group', "1");
|
||||
} else {
|
||||
$pathChild->addChild('is_group', "0");
|
||||
}
|
||||
$pathChild->addChild('name', htmlspecialchars($pathObject->getName()));
|
||||
}
|
||||
|
||||
// turn it into XML
|
||||
$dom = dom_import_simplexml($xml)->ownerDocument;
|
||||
$dom->formatOutput = TRUE;
|
||||
return $dom->saveXML();
|
||||
}, $pbs);
|
||||
|
||||
echo json_encode($xmls);
|
||||
19
internal/env/instances/barrel/wisskiutils/list_uri_prefixes.php
vendored
Normal file
19
internal/env/instances/barrel/wisskiutils/list_uri_prefixes.php
vendored
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
<?php
|
||||
|
||||
/**
|
||||
* This script will list all the URIs that this system is aware of.
|
||||
* This works by listing all the default graph uris of all the adapters.
|
||||
*/
|
||||
|
||||
// iterate over all adapters
|
||||
$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
|
||||
foreach ($storage->loadMultiple() as $adapter) {
|
||||
// read the configuration, and check if we have a default graph
|
||||
$conf = $adapter->getEngine()->getConfiguration();
|
||||
if(!array_key_exists('default_graph', $conf)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// and echo it out
|
||||
echo $conf['default_graph'] . "\n";
|
||||
}
|
||||
13
internal/env/instances/barrel/wisskiutils/set_trusted_host.sh
vendored
Executable file
13
internal/env/instances/barrel/wisskiutils/set_trusted_host.sh
vendored
Executable file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This utility script can be used to configure the trusted host settings inside of settings.php.
|
||||
# It doesn't take care of corner cases and should only be used when needed.
|
||||
|
||||
INSTANCE_DOMAIN="$(hostname -f)"
|
||||
INSTANCE_DOMAIN="${INSTANCE_DOMAIN%.wisski}"
|
||||
|
||||
TRUSTED_HOST_PATTERN="${INSTANCE_DOMAIN//\./\\\\.}"
|
||||
TRUSTED_HOST_PATTERNS='["'$TRUSTED_HOST_PATTERN'"]'
|
||||
|
||||
echo "Setting 'trusted_host_patterns' to $TRUSTED_HOST_PATTERNS"
|
||||
bash /wisskiutils/settings_php_set.sh 'trusted_host_patterns' "$TRUSTED_HOST_PATTERNS"
|
||||
17
internal/env/instances/barrel/wisskiutils/settings_php_get.sh
vendored
Executable file
17
internal/env/instances/barrel/wisskiutils/settings_php_get.sh
vendored
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash
|
||||
|
||||
# settings_php_get.sh name
|
||||
# Gets the 'settings_php_get.php' setting 'name' as json-encoded value, or null when it does not exist.
|
||||
|
||||
NAME=$1
|
||||
|
||||
if [ -z "$NAME" ]; then
|
||||
echo "Usage: get_settings_setting.sh NAME"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
echo "$NAME" | drush php:eval '
|
||||
use \Drupal\Core\Site\Settings;
|
||||
$name=trim(file_get_contents("php://stdin"));
|
||||
echo json_encode(Settings::get($name));
|
||||
';
|
||||
56
internal/env/instances/barrel/wisskiutils/settings_php_set.sh
vendored
Executable file
56
internal/env/instances/barrel/wisskiutils/settings_php_set.sh
vendored
Executable file
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/bash
|
||||
|
||||
# settings_php_set.sh name value
|
||||
# Sets the 'settings.php' setting 'name' to 'value'.
|
||||
# Value must be json-encoded.
|
||||
|
||||
NAME=$1
|
||||
VALUE=$2
|
||||
|
||||
if [ -z "$NAME" ]; then
|
||||
echo "Usage: settings_php_set.sh NAME VALUE"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
if [ -z "$VALUE" ]; then
|
||||
echo "Usage: settings_php_set.sh NAME VALUE"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
cd /var/www/data/project
|
||||
chmod u+w web/sites/default/settings.php
|
||||
|
||||
(echo "$NAME"; echo "$VALUE" ) | drush php:eval '
|
||||
include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
|
||||
|
||||
// read NAME and VALUE from STDIN
|
||||
$content=file_get_contents("php://stdin");
|
||||
$newline=strpos($content, "\n");
|
||||
$name=trim(substr($content, 0, $newline));
|
||||
$jvalue=trim(substr($content, $newline + 1));
|
||||
|
||||
// decode json values
|
||||
$value = @json_decode($jvalue);
|
||||
if ($value === null && json_last_error() !== JSON_ERROR_NONE) {
|
||||
echo "Invalid JSON, cannot update settings.php. \n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
// make parameters to drush_rewrite_settings
|
||||
$settings["settings"][$name] = (object)[
|
||||
"value" => $value,
|
||||
"required" => TRUE,
|
||||
];
|
||||
|
||||
// find the actual settings.php file to rewrite
|
||||
$filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
|
||||
drupal_rewrite_settings($settings, $filename);
|
||||
|
||||
echo "Wrote " . $filename . "\n";
|
||||
return 0;
|
||||
';
|
||||
EXIT=$?
|
||||
|
||||
chmod u-w web/sites/default/settings.php
|
||||
|
||||
exit $?
|
||||
4
internal/env/instances/reserve.env
vendored
Normal file
4
internal/env/instances/reserve.env
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
VIRTUAL_HOST=${VIRTUAL_HOST}
|
||||
|
||||
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
|
||||
26
internal/env/instances/reserve/docker-compose.yml
vendored
Normal file
26
internal/env/instances/reserve/docker-compose.yml
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
static:
|
||||
image: tkw01536/gostatic
|
||||
restart: always
|
||||
environment:
|
||||
# port and hostname for this image to use
|
||||
VIRTUAL_HOST: ${VIRTUAL_HOST}
|
||||
VIRTUAL_PORT: 8043
|
||||
|
||||
# optional letsencrypt email
|
||||
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
|
||||
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
|
||||
|
||||
ports:
|
||||
- 8043
|
||||
|
||||
# volumes that are mounted
|
||||
volumes:
|
||||
- ./index.html:/srv/http/index.html:ro
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
4
internal/env/instances/reserve/index.html
vendored
Normal file
4
internal/env/instances/reserve/index.html
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
This domain name is reserved.
|
||||
Content is a work in progress.
|
||||
90
internal/env/instances_provision.go
vendored
Normal file
90
internal/env/instances_provision.go
vendored
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/config"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func (dis *Distillery) InstancesDir() string {
|
||||
return filepath.Join(dis.Config.DeployRoot, "instances")
|
||||
}
|
||||
|
||||
func (dis *Distillery) InstanceDir(slug string) string {
|
||||
return filepath.Join(dis.InstancesDir(), slug)
|
||||
}
|
||||
|
||||
func (dis *Distillery) InstanceSQL(slug string) (database, user string) {
|
||||
database = dis.Config.MysqlDatabasePrefix + slug
|
||||
user = dis.Config.MysqlUserPrefix + slug
|
||||
return
|
||||
}
|
||||
|
||||
func (dis *Distillery) InstanceGraphDB(slug string) (repo, user string) {
|
||||
repo = dis.Config.GraphDBRepoPrefix + slug
|
||||
user = dis.Config.GraphDBUserPrefix + slug
|
||||
return
|
||||
}
|
||||
|
||||
var errInvalidSlug = errors.New("Not a valid slug")
|
||||
|
||||
// NewInstance fills the struct for a new distillery instance.
|
||||
// It validates that slug is a valid name for an instance.
|
||||
//
|
||||
// It does not perform any checks if the instance already exists, or does the creation in the database.
|
||||
func (dis *Distillery) NewInstance(slug string) (i Instance, err error) {
|
||||
|
||||
// make sure that the slug is valid!
|
||||
if _, err := config.IsValidSlug(slug); err != nil {
|
||||
return i, errInvalidSlug
|
||||
}
|
||||
|
||||
// generate sql data
|
||||
sqlPassword, err := dis.Config.NewPassword()
|
||||
if err != nil {
|
||||
return i, err
|
||||
}
|
||||
sqlDB, sqlUser := dis.InstanceSQL(slug)
|
||||
|
||||
// generate ts data
|
||||
tsPassword, err := dis.Config.NewPassword()
|
||||
if err != nil {
|
||||
return i, err
|
||||
}
|
||||
tsRepo, tsUser := dis.InstanceGraphDB(slug)
|
||||
|
||||
// generate drupal data
|
||||
drPassword, err := dis.Config.NewPassword()
|
||||
if err != nil {
|
||||
return i, err
|
||||
}
|
||||
drUser := "admin"
|
||||
|
||||
// make the instance object!
|
||||
instance := bookkeeping.Instance{
|
||||
Slug: slug,
|
||||
|
||||
OwnerEmail: "",
|
||||
AutoBlindUpdateEnabled: true,
|
||||
|
||||
FilesystemBase: dis.InstanceDir(slug),
|
||||
|
||||
SqlDatabase: sqlDB,
|
||||
SqlUser: sqlUser,
|
||||
SqlPassword: sqlPassword,
|
||||
|
||||
GraphDBRepository: tsRepo,
|
||||
GraphDBUser: tsUser,
|
||||
GraphDBPassword: tsPassword,
|
||||
}
|
||||
|
||||
i.DrupalUsername = drUser
|
||||
i.DrupalPassword = drPassword
|
||||
|
||||
// store the instance in the object and return it!
|
||||
i.Instance = instance
|
||||
i.dis = dis
|
||||
return i, nil
|
||||
}
|
||||
8
internal/env/runtime.go
vendored
Normal file
8
internal/env/runtime.go
vendored
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
package env
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
// RuntimeDir returns the path to the runtime directory
|
||||
func (dis *Distillery) RuntimeDir() string {
|
||||
return filepath.Join(dis.Config.DeployRoot, "runtime")
|
||||
}
|
||||
33
internal/env/server.go
vendored
Normal file
33
internal/env/server.go
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// TODO: Move this into dis!
|
||||
|
||||
// Server represents a server for this distillery
|
||||
type Server struct {
|
||||
dis *Distillery
|
||||
}
|
||||
|
||||
func (dis *Distillery) Server() *Server {
|
||||
return &Server{
|
||||
dis: dis,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
instances, err := s.dis.AllInstances()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
io.WriteString(w, "Something went wrong")
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
for _, instance := range instances {
|
||||
io.WriteString(w, instance.Slug+"\n")
|
||||
}
|
||||
}
|
||||
253
internal/env/snapshot.go
vendored
Normal file
253
internal/env/snapshot.go
vendored
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
package env
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/bookkeeping"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/pkg/password"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// SnapshotsDir returns the path that contains all snapshot related data.
|
||||
func (dis *Distillery) SnapshotsDir() string {
|
||||
return filepath.Join(dis.Config.DeployRoot, "snapshots")
|
||||
}
|
||||
|
||||
// SnapshotsStagingPath returns the path to the directory containing a temporary staging area for snapshots.
|
||||
// Use NewSnapshotStagingDir to generate a new staging area.
|
||||
func (dis *Distillery) SnapshotsStagingPath() string {
|
||||
return filepath.Join(dis.SnapshotsDir(), "staging")
|
||||
}
|
||||
|
||||
// SnapshotsArchivePath returns the path to the directory containing all exported archives.
|
||||
// Use NewSnapshotArchivePath to generate a path to a new archive in this directory.
|
||||
func (dis *Distillery) SnapshotsArchivePath() string {
|
||||
return filepath.Join(dis.SnapshotsDir(), "archives")
|
||||
}
|
||||
|
||||
// NewSnapshotArchivePath returns the path to a new archive with the provided prefix.
|
||||
// The path is guaranteed to not exist.
|
||||
func (dis *Distillery) NewSnapshotArchivePath(prefix string) (path string) {
|
||||
// TODO: Consider moving these into a subdirectory with the provided prefix.
|
||||
for path == "" || fsx.Exists(path) {
|
||||
name := dis.newSnapshotName(prefix) + ".tar.gz"
|
||||
path = filepath.Join(dis.SnapshotsArchivePath(), name)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// newSnapshot name returns a new basename for a snapshot with the provided prefix.
|
||||
// The name is guaranteed to be unique within this process.
|
||||
func (*Distillery) newSnapshotName(prefix string) string {
|
||||
suffix, _ := password.Password(64) // silently ignore any errors!
|
||||
if prefix == "" {
|
||||
prefix = "backup"
|
||||
} else {
|
||||
prefix = "snapshot-" + prefix
|
||||
}
|
||||
return fmt.Sprintf("%s-%d-%s", prefix, time.Now().Unix(), suffix)
|
||||
}
|
||||
|
||||
// NewSnapshotStagingDir returns the path to a new snapshot directory.
|
||||
// The directory is guaranteed to have been freshly created.
|
||||
func (dis *Distillery) NewSnapshotStagingDir(prefix string) (path string, err error) {
|
||||
for path == "" || os.IsExist(err) {
|
||||
path = filepath.Join(dis.SnapshotsStagingPath(), dis.newSnapshotName(prefix))
|
||||
err = os.Mkdir(path, os.ModeDir)
|
||||
}
|
||||
if err != nil {
|
||||
path = ""
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SnapshotDescription is a description for a snapshot
|
||||
type SnapshotDescription struct {
|
||||
Dest string // destination path
|
||||
Keepalive bool // should we keep the instance alive while making the snapshot?
|
||||
}
|
||||
|
||||
// Snapshot represents the result of generating a snapshot
|
||||
type Snapshot struct {
|
||||
Description SnapshotDescription
|
||||
Instance bookkeeping.Instance
|
||||
|
||||
// various error states, which are ignored when creating the snapshot
|
||||
ErrPanic interface{} // panic, if any
|
||||
|
||||
ErrStart error
|
||||
ErrStop error
|
||||
|
||||
ErrBookkeep error
|
||||
ErrPathbuilder error
|
||||
ErrFilesystem error
|
||||
ErrTriplestore error
|
||||
ErrSSQL error
|
||||
}
|
||||
|
||||
// Snapshot creates a new snapshot of this instance into dest
|
||||
func (instance Instance) Snapshot(io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
|
||||
// setup the snapshot
|
||||
snapshot.Description = desc
|
||||
snapshot.Instance = instance.Instance
|
||||
|
||||
// catch anything critical that happened during the snapshot
|
||||
defer func() {
|
||||
snapshot.ErrPanic = recover()
|
||||
}()
|
||||
|
||||
// and do the create!
|
||||
snapshot.create(io, instance)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) {
|
||||
stack := instance.Stack()
|
||||
|
||||
// stop the instance (unless it was explicitly asked to not do so!)
|
||||
if !snapshot.Description.Keepalive {
|
||||
logging.LogMessage(io, "Stopping instance")
|
||||
snapshot.ErrStop = stack.Down(io)
|
||||
defer func() {
|
||||
logging.LogMessage(io, "Starting instance")
|
||||
snapshot.ErrStart = stack.Up(io)
|
||||
}()
|
||||
}
|
||||
|
||||
// create a wait group, and message channel
|
||||
wg := &sync.WaitGroup{}
|
||||
messages := make(chan string, 4)
|
||||
|
||||
// write bookkeeping information
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
bkPath := filepath.Join(snapshot.Description.Dest, "bookkeeping.txt")
|
||||
messages <- bkPath
|
||||
|
||||
info, err := os.Create(bkPath)
|
||||
if err != nil {
|
||||
snapshot.ErrBookkeep = err
|
||||
return
|
||||
}
|
||||
defer info.Close()
|
||||
|
||||
// print whatever is in the database
|
||||
// TODO: This should be sql code, maybe gorm can do that?
|
||||
_, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance)
|
||||
}()
|
||||
|
||||
// write pathbuilders
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders")
|
||||
messages <- pbPath
|
||||
|
||||
// create the directory!
|
||||
if err := os.Mkdir(pbPath, fs.ModeDir); err != nil {
|
||||
snapshot.ErrPathbuilder = err
|
||||
return
|
||||
}
|
||||
|
||||
// put in all the pathbuilders
|
||||
snapshot.ErrPathbuilder = instance.ExportPathbuilders(pbPath)
|
||||
}()
|
||||
|
||||
// backup the filesystem
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
fsPath := filepath.Join(snapshot.Description.Dest, filepath.Base(instance.FilesystemBase))
|
||||
if err := os.Mkdir(fsPath, fs.ModeDir); err != nil {
|
||||
snapshot.ErrFilesystem = err
|
||||
return
|
||||
}
|
||||
|
||||
// copy over whatever is in the base directory
|
||||
snapshot.ErrFilesystem = fsx.CopyDirectory(fsPath, instance.FilesystemBase, func(dst, src string) {
|
||||
messages <- dst
|
||||
})
|
||||
}()
|
||||
|
||||
// backup the graph db repository
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
tsPath := filepath.Join(snapshot.Description.Dest, instance.GraphDBRepository+".nq")
|
||||
messages <- tsPath
|
||||
|
||||
nquads, err := os.Create(tsPath)
|
||||
if err != nil {
|
||||
snapshot.ErrTriplestore = err
|
||||
}
|
||||
defer nquads.Close()
|
||||
|
||||
// directly store the result
|
||||
_, snapshot.ErrTriplestore = instance.dis.Triplestore().Backup(nquads, instance.GraphDBRepository)
|
||||
}()
|
||||
|
||||
// backup the sql database
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
sqlPath := filepath.Join(snapshot.Description.Dest, snapshot.Instance.SqlDatabase+".sql")
|
||||
messages <- sqlPath
|
||||
|
||||
sql, err := os.Create(sqlPath)
|
||||
if err != nil {
|
||||
snapshot.ErrSSQL = err
|
||||
return
|
||||
}
|
||||
defer sql.Close()
|
||||
|
||||
// directly store the result
|
||||
snapshot.ErrSSQL = instance.dis.SQL().Backup(io, sql, instance.SqlDatabase)
|
||||
}()
|
||||
|
||||
// TODO: Backup the docker image
|
||||
|
||||
// wait for the group, then close the message channel.
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(messages)
|
||||
}()
|
||||
|
||||
// print out all the messages
|
||||
for message := range messages {
|
||||
io.Println(message)
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport writes out the report belonging to this snapshot.
|
||||
// It is a separate function, to allow writing it indepenently of the rest.
|
||||
func (snapshot Snapshot) WriteReport(io stream.IOStream) error {
|
||||
return logging.LogOperation(func() error {
|
||||
reportPath := filepath.Join(snapshot.Description.Dest, "report.txt")
|
||||
io.Println(reportPath)
|
||||
|
||||
// create the report file!
|
||||
report, err := os.Create(reportPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer report.Close()
|
||||
|
||||
// print the report into it!
|
||||
_, err = fmt.Fprintf(report, "%#v\n", snapshot)
|
||||
return err
|
||||
}, io, "Writing snapshot report")
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
// Package execx defines extensions to the "os/exec" package
|
||||
package execx
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// ExecCommandError is returned by Exec when a command could not be executed.
|
||||
// This typically hints that the executable cannot be found, but may have other causes.
|
||||
const ExecCommandError = 127
|
||||
|
||||
// Exec executes a system command with the specified input/output streams, working directory, and arguments.
|
||||
//
|
||||
// If the command executes, it's exit code will be returned.
|
||||
// If the command can not be executed, returns [ExecCommandError].
|
||||
func Exec(io stream.IOStream, workdir string, exe string, argv ...string) int {
|
||||
// setup the command
|
||||
cmd := exec.Command(exe, argv...)
|
||||
cmd.Dir = workdir
|
||||
cmd.Stdin = io.Stdin
|
||||
cmd.Stdout = io.Stdout
|
||||
cmd.Stderr = io.Stderr
|
||||
|
||||
// run it
|
||||
err := cmd.Run()
|
||||
|
||||
// non-zero exit
|
||||
if err, ok := err.(*exec.ExitError); ok {
|
||||
return err.ExitCode()
|
||||
}
|
||||
|
||||
// unknown error
|
||||
if err != nil {
|
||||
return ExecCommandError
|
||||
}
|
||||
|
||||
// everything is fine!
|
||||
return 0
|
||||
}
|
||||
|
||||
// MustExec is like Exec, except that it returns true if the command exited successfully, and else false.
|
||||
func MustExec(io stream.IOStream, workdir string, exe string, argv ...string) bool {
|
||||
return Exec(io, workdir, exe, argv...) == 0
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package execx
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// LookPathAbs is like [exec.LookPath], but always returns an absolute path
|
||||
func LookPathAbs(file string) (string, error) {
|
||||
path, err := exec.LookPath(file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Abs(path)
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
package fsx
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
var ErrCopySameFile = errors.New("src and dst must be different files")
|
||||
|
||||
// CopyFile copies a file from src to dst.
|
||||
// When dst and src are the same file, returns ErrCopySameFile.
|
||||
func CopyFile(dst, src string) error {
|
||||
if SameFile(src, dst) {
|
||||
return ErrCopySameFile
|
||||
}
|
||||
|
||||
// open the source
|
||||
srcFile, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
// stat it to get the mode!
|
||||
srcStat, err := srcFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// open or create the destination
|
||||
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, srcStat.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dstFile.Close()
|
||||
|
||||
// and do the copy!
|
||||
_, err = io.Copy(dstFile, srcFile)
|
||||
return err
|
||||
}
|
||||
|
||||
var ErrCopyNoDirectory = errors.New("dst is not a directory")
|
||||
|
||||
// CopyDirectory copies the directory src to dst recursively.
|
||||
// The destination directory must exist, or an error is returned.
|
||||
//
|
||||
// onCopy, when not nil, is called for each file or directory being copied.
|
||||
func CopyDirectory(dst, src string, onCopy func(dst, src string)) error {
|
||||
// sanity checks
|
||||
if SameFile(src, dst) {
|
||||
return ErrCopySameFile
|
||||
}
|
||||
if !IsDirectory(dst) {
|
||||
return ErrCopyNoDirectory
|
||||
}
|
||||
|
||||
// call onCopy for this directory!
|
||||
if onCopy != nil {
|
||||
onCopy(dst, src)
|
||||
}
|
||||
|
||||
// iterate over the entries or bail out
|
||||
entries, err := os.ReadDir(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
eDest := filepath.Join(dst, name)
|
||||
eSrc := filepath.Join(src, name)
|
||||
|
||||
// it is not a directory => Use CopyFile
|
||||
if !entry.IsDir() {
|
||||
if onCopy != nil {
|
||||
onCopy(eDest, eSrc)
|
||||
}
|
||||
|
||||
// do the copy!
|
||||
if err := CopyFile(eDest, eSrc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// find out the mode of the entry
|
||||
eInfo, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// make the target directory
|
||||
if err := os.Mkdir(eDest, eInfo.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do the copy!
|
||||
if err := CopyDirectory(eDest, eSrc, onCopy); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
package fsx
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// SameFile checks if path1 and path2 refer to the same file.
|
||||
// If both files exist, they are compared using [os.SameFile].
|
||||
// If both files do not exist, the paths are first compared syntactically and then via recursion on [filepath.Dir].
|
||||
func SameFile(path1, path2 string) bool {
|
||||
|
||||
// initial attempt: check if directly
|
||||
same, certain := couldBeSameFile(path1, path2)
|
||||
if certain {
|
||||
return same
|
||||
}
|
||||
|
||||
// second attempt: find the directory names and base paths
|
||||
d1, n1 := filepath.Dir(path1), filepath.Base(path1)
|
||||
d2, n2 := filepath.Dir(path2), filepath.Base(path2)
|
||||
|
||||
// if we have different file names (and they don't exist)
|
||||
// we don't need to continue
|
||||
if n1 != n2 {
|
||||
return false
|
||||
}
|
||||
|
||||
// compare the base names!
|
||||
{
|
||||
same, _ := couldBeSameFile(d1, d2)
|
||||
return same
|
||||
}
|
||||
}
|
||||
|
||||
// couldBeSameFile checks if path1 might be the same as path2.
|
||||
//
|
||||
// If both files exist, compares using [os.SameFile].
|
||||
// Otherwise compares absolute paths using string comparison.
|
||||
//
|
||||
// same indicates if they might be the same file.
|
||||
// authorative indiciates if the result is authorative.
|
||||
func couldBeSameFile(path1, path2 string) (same, authorative bool) {
|
||||
{
|
||||
// stat both files
|
||||
info1, err1 := os.Stat(path1)
|
||||
info2, err2 := os.Stat(path2)
|
||||
|
||||
// both files exist => check using os.SameFile
|
||||
// the result is always authorative
|
||||
if err1 == nil && err2 == nil {
|
||||
same = os.SameFile(info1, info2)
|
||||
authorative = true
|
||||
return
|
||||
}
|
||||
|
||||
// only 1 file errored => they could be different
|
||||
if (err1 == nil) != (err2 == nil) {
|
||||
return
|
||||
}
|
||||
|
||||
// only 1 file does not exist => they could be different
|
||||
if os.IsNotExist(err1) != os.IsNotExist(err2) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// resolve paths absolutely
|
||||
rpath1, err1 := filepath.Abs(path1)
|
||||
rpath2, err2 := filepath.Abs(path2)
|
||||
|
||||
// if either path could not be resolved absolutely
|
||||
// fallback to just using clean!
|
||||
if err1 != nil {
|
||||
rpath1 = filepath.Clean(path1)
|
||||
}
|
||||
if err2 != nil {
|
||||
rpath2 = filepath.Clean(path2)
|
||||
}
|
||||
|
||||
// compare using strings
|
||||
same = rpath1 == rpath2
|
||||
authorative = same // positive result is authorative!
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
package fsx
|
||||
|
||||
import (
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Touch touches a file.
|
||||
// It is similar to the unix 'touch' command.
|
||||
//
|
||||
// If the file does not exist exists, it is created using [os.Create].
|
||||
// If the file does exist, it's access and modification times are updated to the current time.
|
||||
func Touch(path string) error {
|
||||
_, err := os.Stat(path)
|
||||
switch {
|
||||
case os.IsNotExist(err):
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
now := time.Now().Local()
|
||||
return os.Chtimes(path, now, now)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
// Package fsx provides convenient abstractions to work with the filesystem.
|
||||
package fsx
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Exists checks if the given path exists
|
||||
func Exists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// IsDirectory checks if the provided path exists and is a directory
|
||||
func IsDirectory(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.Mode().IsDir()
|
||||
}
|
||||
|
||||
// IsFile checks if the provided path exists and is a regular file
|
||||
func IsFile(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
return err == nil && info.Mode().IsRegular()
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
// Package hostname provides the hostname.
|
||||
package hostname
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/Showmax/go-fqdn"
|
||||
)
|
||||
|
||||
// FQDN attempts to return the fully qualified domain name of the host system.
|
||||
// If an error occurs, may fall back to the empty string.
|
||||
func FQDN() string {
|
||||
|
||||
// try the hostname function
|
||||
{
|
||||
fqdn, err := fqdn.FqdnHostname()
|
||||
if err == nil {
|
||||
return fqdn
|
||||
}
|
||||
}
|
||||
|
||||
// fallback to os hostname
|
||||
{
|
||||
hostname, err := os.Hostname()
|
||||
if err == nil {
|
||||
return hostname
|
||||
}
|
||||
}
|
||||
|
||||
// use the empty string
|
||||
return ""
|
||||
}
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
package logging
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
var logLevelMutex sync.Mutex
|
||||
var logLevelMap = make(map[uintptr]int)
|
||||
|
||||
func getIndent(io stream.IOStream) int {
|
||||
logLevelMutex.Lock()
|
||||
defer logLevelMutex.Unlock()
|
||||
|
||||
id, ok := logID(io)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
return logLevelMap[id]
|
||||
}
|
||||
|
||||
func incIndent(io stream.IOStream) int {
|
||||
logLevelMutex.Lock()
|
||||
defer logLevelMutex.Unlock()
|
||||
|
||||
id, ok := logID(io)
|
||||
if !ok { // if we don't have an id, then inc statically returns 1
|
||||
return 1
|
||||
}
|
||||
|
||||
logLevelMap[id]++
|
||||
return logLevelMap[id]
|
||||
}
|
||||
|
||||
func decIndent(io stream.IOStream) int {
|
||||
logLevelMutex.Lock()
|
||||
defer logLevelMutex.Unlock()
|
||||
id, ok := logID(io)
|
||||
|
||||
if !ok { // if we don't have an id, then dec statically returns 0
|
||||
return 0
|
||||
}
|
||||
|
||||
logLevelMap[id]--
|
||||
if logLevelMap[id] < 0 {
|
||||
panic("DecLogIdent: decrease below 0")
|
||||
}
|
||||
return logLevelMap[id]
|
||||
}
|
||||
|
||||
func logID(io stream.IOStream) (uintptr, bool) {
|
||||
file, ok := io.Stdin.(interface{ Fd() uintptr })
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return file.Fd(), true
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package logging
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
// LogOperation logs a message that is displayed to the user, and then increases the log indent level.
|
||||
func LogOperation(operation func() error, io stream.IOStream, format string, args ...interface{}) error {
|
||||
logOperation(io, getIndent(io), format, args...)
|
||||
incIndent(io)
|
||||
defer decIndent(io)
|
||||
|
||||
return operation()
|
||||
}
|
||||
|
||||
// LogMessage logs a message that is displayed to the user
|
||||
func LogMessage(io stream.IOStream, format string, args ...interface{}) (int, error) {
|
||||
return logOperation(io, getIndent(io), format, args...)
|
||||
}
|
||||
|
||||
func logOperation(io stream.IOStream, indent int, format string, args ...interface{}) (int, error) {
|
||||
message := "\033[1m" + strings.Repeat(" ", indent+1) + "=> " + format + "\033[0m\n"
|
||||
if !io.StdinIsATerminal() {
|
||||
message = " => " + format
|
||||
}
|
||||
|
||||
return io.Printf(message, args...)
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
// Package password allows generating random passwords
|
||||
package password
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// NOTE(twiesing): A bunch of scripts cannot properly handle the extra characters in the password.
|
||||
// For now it is disabled, but it should be re-enabled later.
|
||||
const PasswordCharSet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // + "!@#$%&*"
|
||||
var passwordCharCount = big.NewInt(int64(len(PasswordCharSet)))
|
||||
|
||||
// Password returns a randomly generated string with the provided length.
|
||||
// It consists of alphanumeric characters only.
|
||||
//
|
||||
// When an error occurs, it is guaranteed to return "", err.
|
||||
// [rand.Reader] is used as the source of randomness.
|
||||
func Password(length int) (string, error) {
|
||||
if length < 0 {
|
||||
panic("length < 0")
|
||||
}
|
||||
|
||||
// create a buffer to write the string to!
|
||||
var password strings.Builder
|
||||
password.Grow(length)
|
||||
|
||||
for i := 0; i < length; i++ {
|
||||
|
||||
// grab a random bIndex!
|
||||
bIndex, err := rand.Int(rand.Reader, passwordCharCount)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// and use that index!
|
||||
index := int(bIndex.Int64())
|
||||
if err := password.WriteByte(PasswordCharSet[index]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
// return the password!
|
||||
return password.String(), nil
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -1,10 +0,0 @@
|
|||
package sqle
|
||||
|
||||
import (
|
||||
"github.com/feiin/sqlstring"
|
||||
)
|
||||
|
||||
// Format formats the provided query with the given parameters.
|
||||
func Format(query string, params ...interface{}) string {
|
||||
return sqlstring.Format(query, params...)
|
||||
}
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
// Package targz provides facilities for packaging tar.gz files
|
||||
package targz
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Package packages the source directory into a 'tar.gz' file into destination.
|
||||
// If the destination already exists, it is truncated.
|
||||
//
|
||||
// onCopy, when not nil, is called for each file being copied into the archive.
|
||||
func Package(dst, src string, onCopy func(rel string, src string)) (count int64, err error) {
|
||||
// create the target archive
|
||||
archive, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer archive.Close()
|
||||
|
||||
// create a gzip writer
|
||||
zipHandle := gzip.NewWriter(archive)
|
||||
defer zipHandle.Close()
|
||||
|
||||
// create a tar writer
|
||||
tarHandle := tar.NewWriter(zipHandle)
|
||||
defer tarHandle.Close()
|
||||
|
||||
// and walk through it!
|
||||
err = filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// determine the relative path
|
||||
var relpath string
|
||||
relpath, err = filepath.Rel(src, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if onCopy != nil {
|
||||
onCopy(relpath, path)
|
||||
}
|
||||
|
||||
// create a file info header!
|
||||
tInfo, err := tar.FileInfoHeader(info, relpath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tInfo.Name = filepath.ToSlash(relpath)
|
||||
|
||||
// write it!
|
||||
if err := tarHandle.WriteHeader(tInfo); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// a directory => no more writing required
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// open the file
|
||||
handle, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer handle.Close()
|
||||
|
||||
// and copy it into the archive
|
||||
ccount, err := io.Copy(tarHandle, handle)
|
||||
count += ccount
|
||||
return err
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -1,131 +0,0 @@
|
|||
package unpack
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var errExpectedFileButGotDirectory = errors.New("expected a file, but got a directory")
|
||||
var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a file")
|
||||
|
||||
// InstallDir installs the directory at src within fsys to dst.
|
||||
//
|
||||
// onInstallFile is called for each file or directory being installed.
|
||||
//
|
||||
// If the destination path does not exist, it is created using [os.MakeDirs]
|
||||
// The directory is installed recursively.
|
||||
func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
|
||||
// open the source file
|
||||
srcFile, err := fsys.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// stat it!
|
||||
srcInfo, err := srcFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// make sure it's a file!
|
||||
if !srcInfo.IsDir() {
|
||||
return errExpectedDirectoryButGotFile
|
||||
}
|
||||
|
||||
// call the hook (if any)
|
||||
if onInstallFile != nil {
|
||||
onInstallFile(dst, src)
|
||||
}
|
||||
|
||||
// do the installation of the directory.
|
||||
// the type cast should be safe.
|
||||
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
|
||||
}
|
||||
|
||||
// installResource installs the resource at src within fsys to dst.
|
||||
//
|
||||
// OnInstallFile is called for each source and destination file.
|
||||
// OnInstallFile may be nil.
|
||||
func installResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
|
||||
// open the srcFile
|
||||
srcFile, err := fsys.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
// stat it!
|
||||
srcInfo, err := srcFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// call the hook (if any)
|
||||
if onInstallFile != nil {
|
||||
onInstallFile(dst, src)
|
||||
}
|
||||
|
||||
// this is a directory, so the cast is safe!
|
||||
if srcInfo.IsDir() {
|
||||
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
|
||||
}
|
||||
|
||||
// this is a regular file!
|
||||
return installFile(dst, srcInfo, srcFile)
|
||||
}
|
||||
|
||||
func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
|
||||
// create the destination
|
||||
dstStat, dstErr := os.Stat(dst)
|
||||
switch {
|
||||
case os.IsNotExist(dstErr):
|
||||
if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
|
||||
return errors.Wrapf(err, "Error creating destination directory %s", dst)
|
||||
}
|
||||
case dstErr != nil:
|
||||
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
|
||||
case !dstStat.IsDir():
|
||||
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
|
||||
}
|
||||
|
||||
// NOTE(twiesing): We don't use fs.Walk here.
|
||||
// If we did, we'd have to reconstruct relative paths.
|
||||
// That would be very ugly!
|
||||
|
||||
// read the directory
|
||||
entries, err := srcFile.ReadDir(-1)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
|
||||
}
|
||||
|
||||
// iterate over all the children
|
||||
for _, entry := range entries {
|
||||
if err := installResource(
|
||||
filepath.Join(dst, entry.Name()),
|
||||
filepath.Join(src, entry.Name()),
|
||||
fsys,
|
||||
onInstallFile,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func installFile(dst string, srcInfo fs.FileInfo, src fs.File) error {
|
||||
// create the file using the right mode!
|
||||
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// copy over the content!
|
||||
_, err = io.Copy(file, src)
|
||||
return errors.Wrapf(err, "Error writing to destination %s", dst)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue