Do a large chunk of the move to go

This commit moves a huge chunk of the code to go. The TODO.md document
indicates what is left to be done.
This commit is contained in:
Tom Wiesing 2022-08-14 10:57:59 +02:00
parent db2ad9b4bd
commit 7b38fdd801
No known key found for this signature in database
93 changed files with 4689 additions and 645 deletions

View file

@ -1,25 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# if the site doesn't exist, I can't open a shell.
if ! sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' does not exist in bookeeping table. "
echo "I can't rebuild it. "
exit 1
fi;
# Read everything from the database
read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
# cd into the right directory
cd "$INSTANCE_BASE_DIR"
# and open a www-data shell
docker-compose exec barrel /user_shell.sh /utils/blind_update.sh

View file

@ -1,16 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
DISABLE_LOG=0
# update all the instances
for slug in $(sql_bookkeep_list_updateable); do
log_info "=> /bin/bash $DIR/blind_update.sh '$slug'"
/bin/bash "$DIR/blind_update.sh" "$slug";
done

View file

@ -1,18 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
unset DISABLE_LOG
# update all the instances
for slug in $(sql_bookkeep_list); do
read -r INSTANCE_BASE_DIR <<< "$(sql_bookkeep_load "${slug}" "filesystem_base" | tail -n +2)"
log_info "=> Runnning cron for '$slug'"
cd "$INSTANCE_BASE_DIR"
docker-compose exec barrel /bin/bash /utils/cron.sh
done

View file

@ -1,45 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
DISABLE_LOG=0
require_slug_argument
# if the site doesn't exist, I can't open a shell.
if ! sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' does not exist in bookeeping table. "
echo "I can't show info about it. "
exit 1
fi;
# Read everything from the database
read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER GRAPHDB_PASSWORD <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user,graphdb_password" | tail -n +2)"
GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
# read sql configuration
cd "$INSTANCE_BASE_DIR"
read -r SQL_DATABASE SQL_USER SQL_PASS SQL_OTHER <<< "$(docker-compose exec barrel drush sql:conf --format=tsv --show-passwords)"
echo "=================================================================================="
echo "URL: http://$INSTANCE_DOMAIN"
echo "Base directory: ${INSTANCE_BASE_DIR}"
log_info " => Your GraphDB details (for WissKI Salz) are: "
echo "Read URL: http://triplestore:7200/repositories/$GRAPHDB_REPO"
echo "Write URL: http://triplestore:7200/repositories/$GRAPHDB_REPO/statements"
echo "Username: $GRAPHDB_USER"
echo "Password: $GRAPHDB_PASSWORD"
echo "Authorization Header: $GRAPHDB_HEADER"
echo "Writable: yes"
echo "Default Graph URI: http://$INSTANCE_DOMAIN/#"
echo "Ontology Paths: (empty)"
echo "SameAs property: http://www.w3.org/2002/07/owl#sameAs"
log_info " => Your SQL detsils are: "
echo "SQL Database: $SQL_DATABASE"
echo "SQL Username: $SQL_USER"
echo "SQL Password: $SQL_PASS"

View file

@ -1,12 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
DISABLE_LOG=0
# list all the existing instances
sql_bookkeep_list

View file

@ -1,33 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh
DISABLE_LOG=0
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
# wait for sql to come up
wait_for_sql > /dev/null
echo "Creating new MySQL user with root privileges. "
read -p 'Enter Username:' MYSQL_USER
read -sp 'Enter password:' MYSQL_PASSWORD
if ! is_valid_slug "$MYSQL_USER"; then
echo "Not a valid username: ${MYSQL_USER}"
echo "User must be alphanumeric for sql injection reasons. "
echo "You can always create a user manually. "
exit 1
fi
if ! is_valid_slug "$MYSQL_PASSWORD"; then
echo "Not a valid password: ${MYSQL_PASSWORD}"
echo "Password must be alphanumeric for sql injection reasons. "
echo "You can always create a user manually. "
exit 1
fi
dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO \`${MYSQL_USER}\`@\`%\` WITH GRANT OPTION; FLUSH PRIVILEGES;"
log_info "Created user ${MYSQL_USER}"

View file

@ -1,12 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
# wait for sql to come up
wait_for_sql > /dev/null
dockerized_mysql_interactive "$@"

View file

@ -1,106 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# wait for sql to be awake
wait_for_sql
# check if the site exists
if sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' already exists in bookeeping table. "
echo "Refusing to work"
exit 1;
fi
# Randomly generate the database name and user we will configure.
# Use the 'randompw' alias for this.
log_info " => Generating new MySQL password"
MYSQL_PASSWORD="$(randompw)"
# Initialize the SQL database with those credentials.
log_info " => Intializing new SQL database '${MYSQL_DATABASE}' and user '$MYSQL_USER'. "
dockerized_mysql -e "CREATE DATABASE \`${MYSQL_DATABASE}\`;"
dockerized_mysql -e "CREATE USER \`${MYSQL_USER}\`@'%' IDENTIFIED BY '${MYSQL_PASSWORD}';"
dockerized_mysql -e "GRANT ALL PRIVILEGES ON \`${MYSQL_DATABASE}\`.* TO \`${MYSQL_USER}\`@\`%\`;"
dockerized_mysql -e "FLUSH PRIVILEGES;"
# Create a new repository for GraphDB.
# Use the template for this.
log_info " => Generating new GraphDB repository '$GRAPHDB_REPO'"
load_template "repository/graphdb-repo.ttl" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "INSTANCE_DOMAIN" "${INSTANCE_DOMAIN}" | \
curl -X POST $GRAPHDB_AUTH_FLAGS \
http://127.0.0.1:7200/rest/repositories \
--header 'Content-Type: multipart/form-data' \
-F "config=@-"
# Generate a random password for the GraphDB user
log_info " => Generating a new GraphDB password"
GRAPHDB_PASSWORD="$(randompw)"
# Create the user and grant them access to the creatd database.
log_info " => Creating GraphDB user '$GRAPHDB_USER'"
load_template "repository/graphdb-user.json" "GRAPHDB_USER" "${GRAPHDB_USER}" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "GRAPHDB_PASSWORD" "${GRAPHDB_PASSWORD}" | \
curl -X POST $GRAPHDB_AUTH_FLAGS \
"http://127.0.0.1:7200/rest/security/users/${GRAPHDB_USER}" \
--header 'Content-Type: application/json' \
--header 'Accept: text/plain' \
-d @-
log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'"
mkdir -p "$INSTANCE_BASE_DIR"
mkdir -p "$INSTANCE_DATA_DIR"
mkdir -p "$INSTANCE_DATA_DIR/.composer"
mkdir -p "$INSTANCE_DATA_DIR/data"
touch "$INSTANCE_DATA_DIR/authorized_keys"
chmod a+rw "$INSTANCE_DATA_DIR/authorized_keys"
# Generate some more random credentials, this time for drupal.
# We again make use of the randompw alias.
log_info " => Generating new drupal credentials"
DRUPAL_USER="admin"
DRUPAL_PASS="$(randompw)"
# TODO: copy over docker-compose into the right directory
log_info " => Creating instance directory"
install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR"
# Log all the details into the bookeeping database
log_info "=> Storing configuration in bookkeeping table"
sql_bookkeep_insert \
"slug,filesystem_base,sql_database,sql_user,sql_password,graphdb_repository,graphdb_user,graphdb_password" \
"\"${SLUG}\",\"${INSTANCE_BASE_DIR}\",\"${MYSQL_DATABASE}\",\"${MYSQL_USER}\",\"${MYSQL_PASSWORD}\",\"${GRAPHDB_REPO}\",\"${GRAPHDB_USER}\",\"${GRAPHDB_PASSWORD}\""
log_info " => Writing configuration file"
load_template "docker-env/barrel" \
"REAL_PATH" "${INSTANCE_DATA_DIR}" \
"GLOBAL_AUTHORIZED_KEYS_FILE" "${GLOBAL_AUTHORIZED_KEYS_FILE}" \
"VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \
"SLUG" "${SLUG}" \
"LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
"LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
"DISTILLERY_DIR" "${DIR}" \
> "$INSTANCE_BASE_DIR/.env"
log_info " => Running and building image"
cd "$INSTANCE_BASE_DIR"
docker-compose build --pull
docker-compose pull
log_info " => Running provision script"
docker-compose run --rm barrel /bin/bash -c "sudo PATH=\$PATH -u www-data /bin/bash /provision_container.sh \
\"${INSTANCE_DOMAIN}\" \
\"${MYSQL_DATABASE}\" \"${MYSQL_USER}\" \"${MYSQL_PASSWORD}\" \
\"${GRAPHDB_REPO}\" \"${GRAPHDB_USER}\" \"${GRAPHDB_PASSWORD}\" \
\"${DRUPAL_USER}\" \"${DRUPAL_PASS}\" \
\"${DRUPAL_VERSION}\" \"${WISSKI_VERSION}\""
log_info " => Starting container"
docker-compose up -d

View file

@ -1,68 +0,0 @@
# To install a new system:
# This script will provision a new Drupal instance and make it available to apache.
# Usage: sudo ./provision.sh $SLUG
# In case the installation fails, it will bail out and leave you with an incomplete installation.
# To delete an incomplete installation, use the ./purge.sh script, or try fixing the error manually.
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# wait for sql to be awake
wait_for_sql
while true; do
log_info " => I'm about to delete the '$SLUG' site from this system. "
read -p "This can not be undone. Please type 'y' to continue: " yn
case $yn in
[Yy]* ) break;;
* ) echo "Abort. "; exit 1;;
esac
done
# check if the site exists
if ! sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' does not exist in bookeeping table. "
echo "I'll try to cleanup with the current defaults. "
echo "This may or may not work. "
else
# Read all the configuration from the database
log_info " => Reading components from database"
read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
fi
# stop the running system container
if [ -d "$INSTANCE_BASE_DIR" ] ; then
log_info "=> Stopping running system"
cd "$INSTANCE_BASE_DIR"
docker-compose down -v || true
fi;
cd
# delete the mysql database.
log_info " => Deleting MySQL database '$MYSQL_DATABASE' and user '$MYSQL_USER'. "
dockerized_mysql -e "DROP DATABASE IF EXISTS \`${MYSQL_DATABASE}\`;"
dockerized_mysql -e "DROP USER IF EXISTS \`${MYSQL_USER}\`@\`%\`;"
dockerized_mysql -e "FLUSH PRIVILEGES;"
# Clear the GraphDB repository.
log_info " => Deleting GraphDB repository '$GRAPHDB_REPO'"
curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/repositories/$GRAPHDB_REPO
log_info " => Deleting GraphDB user '$GRAPHDB_USER'"
curl $GRAPHDB_AUTH_FLAGS -X DELETE http://127.0.0.1:7200/rest/security/users/$GRAPHDB_USER
# Delete the directory
log_info " => Deleting '$INSTANCE_BASE_DIR'"
rm -rf "$INSTANCE_BASE_DIR"
log_info " => Clearing bookkeeping record"
sql_bookeep_delete "$SLUG" || true
log_info " => '$SLUG' has been purged. "

View file

@ -1,29 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# if the site doesn't exist, I can't open a shell.
if ! sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' does not exist in bookeeping table. "
echo "I can't rebuild it. "
exit 1
fi;
# Read everything from the database
read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
log_info " => Touching authorized_keys file"
touch "$INSTANCE_BASE_DIR/data/authorized_keys"
chmod a+rw "$INSTANCE_BASE_DIR/data/authorized_keys"
log_info " => Updating compose files"
install_resource_dir "compose/barrel" "$INSTANCE_BASE_DIR"
log_info "=> Rebuilding and restarting '$INSTANCE_BASE_DIR'"
update_stack "$INSTANCE_BASE_DIR"

View file

@ -1,16 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
DISABLE_LOG=0
# update all the instances
for slug in $(sql_bookkeep_list); do
log_info "=> /bin/bash $DIR/rebuild.sh '$slug'"
/bin/bash "$DIR/rebuild.sh" "$slug";
done

View file

@ -1,41 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# wait for sql to be awake
wait_for_sql
# check if the site exists
if sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' already exists in bookeeping table. "
echo "Refusing to work"
exit 1;
fi
log_info " => Creating local directory structure at '$INSTANCE_BASE_DIR'"
mkdir -p "$INSTANCE_BASE_DIR"
install_resource_dir "compose/reserve" "$INSTANCE_BASE_DIR"
log_info " => Writing configuration file"
load_template "docker-env/reserve" \
"VIRTUAL_HOST" "${INSTANCE_DOMAIN}" \
"SLUG" "${SLUG}" \
"LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
"LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
> "$INSTANCE_BASE_DIR/.env"
log_info " => Running and building image"
cd "$INSTANCE_BASE_DIR"
docker-compose build --pull
docker-compose pull
log_info " => Starting container"
docker-compose up -d
log_info " => $INSTANCE_DOMAIN has been reserved"

120
distillery/resources.go Normal file
View file

@ -0,0 +1,120 @@
// TODO: Rename this to resources oncen finished
package distillery
import (
"embed"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// resourceEmbed contains all the resources required by the WissKI-Distillery package.
//go:embed all:resources
var resourceEmbed embed.FS
// InstallResource install a resource src into dest.
// When it encounters a directory, recursively installs the directory is called.
// For each installation item, onInstallFile is called, unless onInstallFile is nil.
//
// If src points to a file, dst must either be an existing file, or not exist.
// If src points to a directory, dst must either be an existing directory, or not exist.
func InstallResource(dst, src string, onInstallFile func(dst, src string)) error {
return installFile(dst, resourceEmbed, src, onInstallFile)
}
var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory")
var errExpectedDirectoryButGotFile = errors.New("Expected a directory, but got a file")
func installFile(dst string, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// call the on-install file path
if onInstallFile != nil {
onInstallFile(dst, src)
}
// open the source file!
srcFile, err := fsys.Open(src)
if err != nil {
return errors.Wrapf(err, "Error opening source file %s", src)
}
defer srcFile.Close()
// stat the source file to install
srcStat, srcErr := srcFile.Stat()
if srcErr != nil {
return errors.Wrapf(srcErr, "Error calling stat on source %s", src)
}
// if it is a directory, we should recurse!
if srcStat.IsDir() {
return installDir(dst, srcStat, srcFile, fsys, src, onInstallFile)
}
// determine if we need to create the destination file, or if it already exists
dstStat, dstErr := os.Stat(dst)
flag := os.O_WRONLY
switch {
case os.IsNotExist(dstErr):
flag |= os.O_CREATE
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case dstStat.IsDir():
return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
}
// Open the file
dstFile, err := os.OpenFile(dst, flag, srcStat.Mode())
if err != nil {
return errors.Wrapf(err, "Error opening destination %s", dst)
}
defer dstFile.Close()
// copy over the content
_, err = io.Copy(dstFile, srcFile)
return errors.Wrapf(err, "Error writing to destination %s", dst)
}
func installDir(dst string, srcStat fs.FileInfo, srcFile fs.File, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// make sure it is a directory!
dir, ok := srcFile.(fs.ReadDirFile)
if !ok {
return errExpectedDirectoryButGotFile
}
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcStat.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case !dstStat.IsDir():
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
case dstErr == nil:
}
// read the directory
entries, err := dir.ReadDir(-1)
if err != nil {
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
}
// iterate over all the children
for _, entry := range entries {
if err := func(dst, src string) error {
return installFile(dst, fsys, src, onInstallFile)
}(
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
); err != nil {
return err
}
}
return nil
}

View file

@ -3,8 +3,8 @@
#######################
# Real path for volumes to be stored
REAL_PATH=/var/www/example.slug
DISTILLERY_DIR=/distillery/
REAL_PATH=/var/www/deploy/instances/example.slug
UTILS_DIR=/var/www/deploy/runtime/utils/
#######################
### Web Server settings

View file

@ -92,7 +92,7 @@ VOLUME /var/www/data
# Add and configure the entrypoint
ADD scripts/entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
CMD ["apache2-foreground"]
# Add the provision script and WissKI utils

View file

@ -25,7 +25,7 @@ services:
- ${REAL_PATH}/.composer:/var/www/.composer
- ${REAL_PATH}/data:/var/www/data
- ${REAL_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
- ${DISTILLERY_DIR}/utils:/utils:ro
- ${UTILS_DIR}:/utils:ro
networks:
default:

View file

@ -170,3 +170,5 @@ function printdetails() {
echo "Password: $DRUPAL_PASS"
}
printdetails
exit 0

View file

@ -1,19 +0,0 @@
#######################
# Meta Settings
#######################
# The target path to redirect to
TARGET=https://github.com/FAU-CDI/wisski-distillery
# path to .json
SELF_OVERRIDES_FILE=/overrides.json
#######################
### Web Server settings
#######################
# the hostname for the website
VIRTUAL_HOST=example.com
# optional letsencrypt support
# when blank, ignore
LETSENCRYPT_HOST=
LETSENCRYPT_EMAIL=

View file

@ -1,11 +1,11 @@
# Several docker-compose files are created to manage global services and the system itself.
# On top of this all real-system space will be created under this directory.
DEPLOY_ROOT=/var/www/deploy
DEPLOY_ROOT=${DEPLOY_ROOT}
# Each created Drupal Instance corresponds to a single domain name.
# These domain names should either be a complete domain name or a sub-domain of a default domain.
# This setting configures the default domain-name to create subdomains of.
DEFAULT_DOMAIN=localhost.kwarc.info
DEFAULT_DOMAIN=${DEFAULT_DOMAIN}
# By default, the default domain redirects to the distillery repository.
# If you want to change this, set an alternate domain name here.
@ -17,7 +17,7 @@ SELF_EXTRA_DOMAINS=
# You can override individual URLS in the homepage.
# Do this by adding URLs (without trailing '/'s) into a JSON file
SELF_OVERRIDES_FILE=/distillery/overrides.json
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
# The system can support setting up certificate(s) automatically.
# It can be enabled by setting an email for certbot certificates.
@ -52,7 +52,12 @@ DISTILLERY_BOOKKEEPING_TABLE=distillery
PASSWORD_LENGTH=64
# A file to be used for global authorized_keys for the ssh server.
GLOBAL_AUTHORIZED_KEYS_FILE=/distillery/authorized_keys
GLOBAL_AUTHORIZED_KEYS_FILE=${AUTHORIZED_KEYS_FILE}
# The admin password of the GraphDB interface, to be used for queries
GRAPHDB_ADMIN_PASSWORD=root
# The admin user and password of the GraphDB interface, to be used for queries
GRAPHDB_ADMIN_USER=${GRAPHDB_ADMIN_USER}
GRAPHDB_ADMIN_PASSWORD=${GRAPHDB_ADMIN_PASSWORD}
# The admin password to use for access to mysql
MYSQL_ADMIN_USER=${MYSQL_ADMIN_USER}
MYSQL_ADMIN_PASSWORD=${MYSQL_ADMIN_PASSWORD}

View file

@ -0,0 +1,2 @@
# This file contains authorized_keys files valid for every repository in the distillery.
# To add a key, add one file per line.

View file

@ -0,0 +1 @@
{}

View file

@ -1,5 +1,5 @@
REAL_PATH=${REAL_PATH}
DISTILLERY_DIR=${DISTILLERY_DIR}
UTILS_DIR=${UTILS_DIR}
SLUG=${SLUG}
VIRTUAL_HOST=${VIRTUAL_HOST}

View file

@ -0,0 +1,112 @@
package distillery
import (
"io"
"io/fs"
"os"
"regexp"
"strings"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
var templateRegexp = regexp.MustCompile(`\${[^}]+}`)
// InstallTemplates open the resource src, and installs it into dst.
// the template resource must fit into memory.
//
// For each variable ${THING} inside dest, a key 'THING' must exist in context.
// Extra or missing template keys are an error.
func InstallTemplate(dst, src string, context map[string]string) error {
bytes, srcMode, err := doTemplate(src, context)
if err != nil {
return err
}
// determine if we need to create the destination file, or if it already exists
dstStat, dstErr := os.Stat(dst)
flag := os.O_WRONLY
switch {
case os.IsNotExist(dstErr):
flag |= os.O_CREATE
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case dstStat.IsDir():
return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
}
// open and write the destination file
dstFile, err := os.OpenFile(dst, flag, srcMode)
if err != nil {
return errors.Wrapf(err, "Unable to open file %s", dst)
}
_, err = dstFile.Write(bytes)
return errors.Wrapf(err, "Unable to write destination %s", dst)
}
// ReadTemplate is like InstallTemplate, except that it writes template into a byte slice and returns it.
func ReadTemplate(src string, context map[string]string) ([]byte, error) {
bytes, _, err := doTemplate(src, context)
return bytes, err
}
func doTemplate(src string, context map[string]string) (bytes []byte, mode fs.FileMode, err error) {
// open the source file!
srcFile, err := resourceEmbed.Open(src)
if err != nil {
return nil, mode, errors.Wrapf(err, "Error opening source file %s", src)
}
defer srcFile.Close()
// stat the source file to install
srcStat, srcErr := srcFile.Stat()
if srcErr != nil {
return nil, mode, errors.Wrapf(srcErr, "Error calling stat on source %s", src)
}
// should not be a directory
if srcStat.IsDir() {
return nil, mode, errors.Wrapf(errExpectedFileButGotDirectory, "Error calling stat on source %s", src)
}
// read the template and replace
templates, err := io.ReadAll(srcFile)
if err != nil {
return nil, mode, errors.Wrapf(err, "Unable to read src file %s", src)
}
// keep track of context keys that have not been used
unuusedContext := make(map[string]struct{}, len(context))
for key := range context {
unuusedContext[key] = struct{}{}
}
// replace the template regexp
// keeping track of unuused errors
var hadError error
templates = templateRegexp.ReplaceAllFunc(templates, func(b []byte) []byte {
name := string(b[2 : len(b)-1]) // remove the leading ${ and trailing }
delete(unuusedContext, name) // mark the key as having been read
value, ok := context[name]
if hadError != nil && !ok {
hadError = errors.Errorf("key %s missing in context", name)
}
return []byte(value)
})
if hadError != nil {
return nil, mode, hadError
}
if len(unuusedContext) != 0 {
keys := maps.Keys(unuusedContext)
slices.Sort(keys)
return nil, mode, errors.Errorf("additional keys %s in context", strings.Join(keys, ","))
}
// return the data and the mode!
return templates, srcStat.Mode(), nil
}

View file

@ -1,27 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh and read the slug argument.
DISABLE_LOG=1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
DISABLE_LOG=0
require_slug_argument
# if the site doesn't exist, I can't open a shell.
if ! sql_bookkeep_exists "$SLUG"; then
log_error "=> Site '$SLUG' does not exist in bookeeping table. "
echo "I can't open a shell there. "
exit 1
fi;
# Read everything from the database
read -r INSTANCE_BASE_DIR MYSQL_DATABASE MYSQL_USER GRAPHDB_REPO GRAPHDB_USER <<< "$(sql_bookkeep_load "${SLUG}" "filesystem_base,sql_database,sql_user,graphdb_repository,graphdb_user" | tail -n +2)"
# cd into the right directory
cd "$INSTANCE_BASE_DIR"
# and open a www-data shell
docker-compose exec barrel /user_shell.sh

View file

@ -1,133 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
# Read the 'GRAPHDB_ZIP' argument from the command line.
# If it's not set, throw an error.
GRAPHDB_ZIP=$1
if [ -z "$GRAPHDB_ZIP" ]; then
log_error "Usage: system_install.sh GRAPHDB_ZIP"
exit 1;
fi;
# print some general info on the screen
log_info "=> Preparing system to become a WissKI Distillery"
echo "This script will install or upgrade this system to become a WissKI distillery. "
echo "It is idempotent and can safely be run multiple times. "
sleep 5
# Install default system upgrades.
log_info "=> Installing system updates"
apt-get update
apt-get upgrade -y
# install docker dependencies.
log_info "=> Installing docker installer dependencies"
apt-get update
apt-get install -y curl
# install docker using an automated script.
log_info "=> Installing docker"
curl -fsSL https://get.docker.com -o - | /bin/sh
# install docker-compose dependencies.
log_info "=> Install docker-compose installer dependencies"
apt-get update
apt-get install -y python3-pip libffi-dev
# install docker-compose.
log_info "=> Installing docker-compose"
pip3 install --upgrade docker-compose
log_info "=> Creating docker-compose directories and files"
mkdir -p "$DEPLOY_INSTANCES_DIR"
mkdir -p "$DEPLOY_WEB_DIR"
mkdir -p "$DEPLOY_SELF_DIR"
mkdir -p "$DEPLOY_RESOLVER_DIR"
mkdir -p "$DEPLOY_SSH_DIR"
mkdir -p "$DEPLOY_TRIPLESTORE_DIR"
mkdir -p "$DEPLOY_SQL_DIR"
mkdir -p "$DEPLOY_BACKUP_INPROGRESS_DIR"
mkdir -p "$DEPLOY_BACKUP_FINAL_DIR"
log_info "=> Creating 'distillery' network"
docker network create distillery || true
log_info "=> Creating 'docker-compose' files for the 'web'. "
install_resource_dir "compose/web" "$DEPLOY_WEB_DIR"
log_info " => Writing 'web' configuration file"
load_template "docker-env/web" \
"DEFAULT_HOST" "${DEFAULT_DOMAIN}" \
> "$DEPLOY_WEB_DIR/.env"
log_info "=> Creating 'docker-compose' files for the 'self'. "
install_resource_dir "compose/self" "$DEPLOY_SELF_DIR"
log_info "=> Creating 'docker-compose' files for the 'resolver'. "
install_resource_dir "compose/resolver" "$DEPLOY_RESOLVER_DIR"
touch "$DEPLOY_PREFIX_CONFIG"
log_info "=> Creating 'docker-compose' files for the 'ssh'. "
install_resource_dir "compose/ssh" "$DEPLOY_SSH_DIR"
# setup the lesencrypt host for the default domain
if [ -n "$LETSENCRYPT_HOST" ]; then
LETSENCRYPT_HOST="$SELF_DOMAIN_SPEC"
fi;
log_info " => Writing 'self' configuration file"
load_template "docker-env/self" \
"VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \
"LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
"LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
"TARGET" "${SELF_REDIRECT}" \
"OVERRIDES_FILE" "${SELF_OVERRIDES_FILE}" \
> "$DEPLOY_SELF_DIR/.env"
log_info " => Writing 'resolver' configuration file"
load_template "docker-env/resolver" \
"VIRTUAL_HOST" "${SELF_DOMAIN_SPEC}" \
"LETSENCRYPT_HOST" "${LETSENCRYPT_HOST}" \
"LETSENCRYPT_EMAIL" "${LETSENCRYPT_EMAIL}" \
"PREFIX_FILE" "${DEPLOY_PREFIX_CONFIG}" \
"DEFAULT_DOMAIN" "${DEFAULT_DOMAIN}" \
"LEGACY_DOMAIN" "${SELF_EXTRA_DOMAINS}" \
> "$DEPLOY_RESOLVER_DIR/.env"
# copy over the directory
log_info "=> Creating 'docker-compose' files for the 'triplestore'. "
install_resource_dir "compose/triplestore" "$DEPLOY_TRIPLESTORE_DIR"
# copy the graphdb.zip
echo "Writing \"$DEPLOY_TRIPLESTORE_DIR/graphdb.zip\""
cp "$GRAPHDB_ZIP" "$DEPLOY_TRIPLESTORE_DIR/graphdb.zip"
# create data (volume) location
mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/data/"
mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/work/"
mkdir -p "$DEPLOY_TRIPLESTORE_DIR/data/logs/"
# copy over the sql resource directory, then ensure the data diretory for sql exists.
log_info "=> Creating 'docker-compose' files for the 'sql'. "
install_resource_dir "compose/sql" "$DEPLOY_SQL_DIR"
mkdir -p "$DEPLOY_SQL_DIR/data/"
# Run all the updates via system_update.sh
log_info " => Running 'system_update.sh'"
bash "$SCRIPT_DIR/system_update.sh"
log_info "=> Waiting for sql to come up"
wait_for_sql
log_info "=> Creating '$DISTILLERY_BOOKKEEPING_DATABASE' database and '$DISTILLERY_BOOKKEEPING_TABLE' table"
load_template "bookkeeping/create.sql" "DATABASE" "$DISTILLERY_BOOKKEEPING_DATABASE" "TABLE" "$DISTILLERY_BOOKKEEPING_TABLE" | \
dockerized_mysql
log_info "=> System installation finished, ready to distill. "

View file

@ -1,34 +0,0 @@
#!/bin/bash
set -e
# read the lib/shared.sh
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
source "$DIR/lib/lib.sh"
log_info "=> Rebuilding and restarting 'web' stack"
update_stack "$DEPLOY_WEB_DIR"
log_info "=> Rebuilding and restarting 'self' stack"
update_stack "$DEPLOY_SELF_DIR"
# build and start the ssh server
log_info "=> Rebuilding and restarting 'ssh' stack"
update_stack "$DEPLOY_SSH_DIR"
# build and start the triplestore
log_info "=> Rebuilding and restarting 'triplestore' stack"
update_stack "$DEPLOY_TRIPLESTORE_DIR"
# build and start the triplestore
log_info "=> Rebuilding and restarting 'sql' stack"
update_stack "$DEPLOY_SQL_DIR"
log_info " => Updating Prefix Config"
cd "$DIR"
bash update_prefix_config.sh
log_info "=> Rebuilding and restarting 'resolver' stack"
update_stack "$DEPLOY_RESOLVER_DIR"
log_info "=> System up-to-date. "