Switch to using Docker

This commit refactors all code in this project to make use of docker.
This has not yet been documented properly.

distillery/resources/.dockerignore

@@ -0,0 +1 @@
+data/

distillery/resources/compose/runtime/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore everything
+*
+
+# allow the following files:
+!conf/*
+!scripts/*

distillery/resources/compose/runtime/.env.sample

@@ -0,0 +1,27 @@
+#######################
+# Meta Settings
+#######################
+
+# Real path for volumes to be stored
+REAL_PATH=/var/www/example.slug
+
+#######################
+### Web Server settings
+#######################
+# the hostname for the website
+VIRTUAL_HOST=example.com
+
+# optional letsencrypt support
+# when blank, ignore
+LETSENCRYPT_HOST=
+LETSENCRYPT_EMAIL=
+
+### SQL settings
+MYSQL_HOST=mysql
+MYSQL_USER=user
+MYSQL_PASS=pass
+
+### GraphDB settings
+GRAPHDB_HOST=graphdb
+GRAPHDB_USER=user
+GRAPHDB_PASS=pass

distillery/resources/compose/runtime/Dockerfile

@@ -0,0 +1,75 @@
+FROM php:7-apache-buster
+WORKDIR /var/www
+
+# install and enable the various required php extension
+RUN apt-get update && apt-get install -y \
+        libcurl4-openssl-dev curl \
+        libpng-dev \
+        libicu-dev \
+        libxml2-dev \
+        libssh2-1-dev \
+        sudo \
+        zip unzip \
+        default-mysql-client \
+    && \
+    docker-php-source extract && \
+    docker-php-ext-install \
+        curl \
+        gd \
+        intl \
+        soap \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        xml \
+        xmlrpc \
+    && \
+    pecl install ssh2-1.2 && \
+    docker-php-ext-enable \
+        curl \
+        gd \
+        intl \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        soap \
+        ssh2 \
+        mysqli \
+        xml \
+        xmlrpc \
+    && \
+    docker-php-source delete
+
+# enable the apache rewrite mod
+RUN a2enmod rewrite
+
+# install composer and add it to path
+RUN curl -sS https://getcomposer.org/installer | php && \
+    mv composer.phar /usr/local/bin/composer
+ENV PATH "/usr/local/bin:/var/www/data/project/vendor/bin:$PATH"
+
+# remove default configuration
+RUN rm /etc/apache2/sites-available/*.conf && \
+    rm /etc/apache2/sites-enabled/*.conf
+# Add wisski configuration
+ADD conf/ports.conf /etc/apache2/ports.conf
+ADD conf/wisski.conf /etc/apache2/sites-available/wisski.conf
+RUN a2ensite wisski
+
+# volumes for composer
+VOLUME /var/www/.composer
+VOLUME /var/www/data
+
+# increase the php memory limit to 2g
+RUN echo 'memory_limit=2G' > /usr/local/etc/php/conf.d/memory-limit.ini
+
+# Add and configure the entrypoint
+ADD scripts/entrypoint.sh /entrypoint.sh
+ENTRYPOINT [ "/entrypoint.sh" ]
+CMD ["apache2-foreground"]
+
+# Add the provision script
+ADD scripts/provision_container.sh /provision_container.sh
+
+# expose port 8080
+EXPOSE 8080

distillery/resources/compose/runtime/conf/ports.conf

@@ -0,0 +1,4 @@
+# This file configures where apache should listen. 
+# Because we are running as a limited user, we want to listen on a high port. 
+# For this we use port 8080
+Listen 8080

distillery/resources/compose/runtime/conf/wisski.conf

@@ -0,0 +1,22 @@
+<VirtualHost *:8080>
+    # the document root -- /var/www/data/project/web
+    DocumentRoot /var/www/data/project/web
+
+    <Directory /var/www/data/project/web>
+        # add types for .owl and .rdf
+        AddType application/rdf+xml .owl
+        AddType application/rdf+xml .rdf
+
+        # Rewrite the 'ontology' directory
+        ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,L]
+        ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [L]
+
+        # Allow overrides of symlinks
+        Options Indexes FollowSymLinks
+        AllowOverride All
+        Require all granted
+    </Directory>
+
+    ErrorLog /dev/stderr
+    CustomLog /dev/stdout combined
+</VirtualHost>

distillery/resources/compose/runtime/docker-compose.yml

@@ -0,0 +1,24 @@
+version: "3.7"
+
+services:
+  runtime:
+    build: .
+    restart: always
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8080
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+    
+    # the volumes to 
+    volumes:
+      - ${REAL_PATH}/.composer:/var/www/.composer
+      - ${REAL_PATH}/data:/var/www/data
+
+networks:
+  default:
+    external:
+      name: distillery

distillery/resources/compose/runtime/scripts/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script contains 
+
+# chown the volumes to make sure they can be read and written by the limited user
+chown www-data:www-data /var/www
+chown www-data:www-data /var/www/.composer
+chown www-data:www-data /var/www/data/
+
+# run the original entrypoint
+docker-php-entrypoint "$@"

distillery/resources/compose/runtime/scripts/provision_container.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+set -e
+
+function log_info() {
+   echo -e "\033[1m$1\033[0m"
+}
+
+function log_ok() {
+   echo -e "\033[0;32m$1\033[0m"
+}
+
+log_info " => Reading configuration variables"
+
+INSTANCE_DOMAIN="$1"
+echo "INSTANCE_DOMAIN=$INSTANCE_DOMAIN"
+shift 1
+
+MYSQL_DATABASE="$1"
+echo "MYSQL_DATABASE=$MYSQL_DATABASE"
+MYSQL_USER="$2"
+echo "MYSQL_USER=$MYSQL_USER"
+MYSQL_PASSWORD="$3"
+echo "MYSQL_PASSWORD=$MYSQL_PASSWORD"
+
+shift 3
+
+GRAPHDB_REPO="$1"
+echo "GRAPHDB_REPO=$GRAPHDB_REPO"
+GRAPHDB_USER="$2"
+echo "GRAPHDB_USER=$GRAPHDB_USER"
+GRAPHDB_PASSWORD="$3"
+echo "GRAPHDB_PASSWORD=$GRAPHDB_PASSWORD"
+shift 3
+
+DRUPAL_USER="$1"
+echo "DRUPAL_USER=$DRUPAL_USER"
+DRUPAL_PASS="$2"
+echo "DRUPAL_PASS=$DRUPAL_PASS"
+shift 2
+
+USE_DRUPAL_9="$1"
+echo "USE_DRUPAL_9=$USE_DRUPAL_9"
+shift 1
+
+log_info " => Preparing installation environment"
+BASE_DIR="/var/www/data"
+COMPOSER_DIR="$BASE_DIR/project"
+WEB_DIR="$COMPOSER_DIR/web"
+ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
+
+log_info " => Creating '$COMPOSER_DIR'"
+mkdir -p "$COMPOSER_DIR"
+cd "$COMPOSER_DIR"
+
+function drupal_sites_permission_workaround() {
+    chmod -R u+w "$WEB_DIR/sites/" || true
+}
+
+# Create a new composer project. 
+log_info " => Creating composer project"
+if [ -z "${USE_DRUPAL_9}" ]; then
+    composer create-project 'drupal/recommended-project:^8.9.0' .
+else
+    composer create-project 'drupal/recommended-project:^9.0.0' .
+fi
+
+# Install drush so that we can automate a lot of things
+log_info " => Installing 'drush'"
+composer require drush/drush
+
+# Use 'drush' to run the site-installation. 
+# Here we need to use the username, password and database creds we made above. 
+log_info " => Running drupal installation scripts"
+drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
+    --account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
+    --db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@sql/${MYSQL_DATABASE}
+drupal_sites_permission_workaround
+
+# create a directory for ontologies. 
+log_info " => Creating '$ONTOLOGY_DIR'"
+mkdir -p "$ONTOLOGY_DIR"
+
+# Install the Wisski packages. 
+log_info " => Installing Wisski packages"
+cd "$COMPOSER_DIR"
+
+# install the development version when requested
+if [ -z "${USE_DRUPAL_9}" ]; then
+    composer require 'drupal/wisski'
+else
+    composer require 'drupal/wisski:2.x-dev'
+fi
+
+drupal_sites_permission_workaround
+composer require drupal/inline_entity_form
+
+drupal_sites_permission_workaround
+composer require drupal/imagemagick
+
+drupal_sites_permission_workaround
+composer require drupal/image_effects
+
+drupal_sites_permission_workaround
+composer require drupal/colorbox
+
+log_info " => Enable Wisski modules"
+drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
+drupal_sites_permission_workaround
+
+log_info " => Provisioning is now complete. "
+log_ok "Your installation details are as follows:"
+function printdetails() {
+    echo "URL:                  http://$INSTANCE_DOMAIN"
+    echo "Username:             $DRUPAL_USER"
+    echo "Password:             $DRUPAL_PASS"
+    log_info " => Your GraphDB details (for WissKI Salz) are: "
+    echo "Read URL:             http://triplestore:7200/repositories/$GRAPHDB_REPO"
+    echo "Write URL:            http://triplestore:7200/repositories/$GRAPHDB_REPO/statements"
+    echo "Username:             $GRAPHDB_USER"
+    echo "Password:             $GRAPHDB_PASSWORD"
+    echo "Writable:             yes"
+    echo "Default Graph URI:    http://$INSTANCE_DOMAIN/#"
+    echo "Ontology Paths:       (empty)"
+    echo "SameAs property:      http://www.w3.org/2002/07/owl#sameAs"
+}
+printdetails

distillery/resources/compose/sql/docker-compose.yml

@@ -0,0 +1,32 @@
+version: "3.7"
+
+services:
+    sql:
+        image: mariadb
+        volumes:
+            - "./data/:/var/lib/mysql"
+        environment:
+            # This combination of environment variables will configure a passwordless root user
+            # that can only connect to the container from 'localhost'.
+            # This means we can only connect using 'docker-compose exec sql mysql -C '...' '. 
+            - "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
+            - "MYSQL_ROOT_HOST=localhost"
+        restart: always
+    phpmyadmin:
+        image: phpmyadmin/phpmyadmin
+        environment:
+            - "PMA_HOST=sql"
+            - "HIDE_PHP_VERSION=true"
+        # phpmyadmin running on localhost:8080 so that we can easily access the system graphically. 
+        # By default no admin account is created, so initial shell access to make one is needed. 
+        ports:
+            - 127.0.0.1:8080:80
+        depends_on:
+            - sql
+        restart: always
+
+
+networks:
+    default:
+        external:
+          name: distillery

distillery/resources/compose/triplestore/Dockerfile

@@ -0,0 +1,60 @@
+# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container. 
+# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
+# but has been modified for performance and security. 
+
+# This image is intended to be built like:
+#   docker build --build-arg graphdb_src=graphdb.zip .
+
+# We first make a base image to base further builds on. 
+# We don't use alpine here, as that uses significantly slower musl instead of glibc. 
+FROM adoptopenjdk/openjdk11:debian-slim as base
+
+# Create a user called graphdb
+RUN useradd -ms /bin/bash graphdb
+
+# make a base images, to add the sources to. 
+FROM base as sources
+
+# install unzip
+RUN apt-get update && apt-get install -y unzip
+
+# add the source file (by default graphdb.zip) to the image
+ARG src=graphdb.zip
+ADD ${src} /graphdb.zip
+
+# unpack it into a temporary directory
+RUN unzip "$src" -d "/unpack/"
+
+# Move it into /opt/graphdb, and chown it to graphdb
+RUN mv "/unpack"/* /opt/graphdb
+RUN chown -R graphdb:graphdb /opt/graphdb
+
+# finally make an image that will run
+FROM base as final
+
+# add the entrypoint script
+ADD entrypoint.sh /entrypoint.sh
+
+# copy over the sources
+COPY --from=sources /opt/graphdb /opt/graphdb
+
+# set environment variables for graphdb_home and path
+ENV GRAPHDB_HOME=/opt/graphdb
+ENV PATH=$GRAPHDB_HOME/bin:$PATH
+
+# expose a port
+EXPOSE 7200
+
+# setup a healthcheck, that checks if the server is up. 
+RUN apt-get update && apt-get install -y curl
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
+
+# Add volumes for data, work and logs as these might be accessible from the outside. 
+# To add your own configuration, manually mount a config file into /opt/graphdb/work
+VOLUME /opt/graphdb/data
+VOLUME /opt/graphdb/work
+VOLUME /opt/graphdb/logs
+
+# setup command and entrypoint
+CMD ["-Dgraphdb.home=/opt/graphdb"]
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

distillery/resources/compose/triplestore/docker-compose.yml

@@ -0,0 +1,17 @@
+version: "3.7"
+
+services:
+    triplestore:
+        build: .
+        ports:
+            - "127.0.0.1:7200:7200"
+        volumes:
+            - './data/data:/opt/graphdb/data'
+            - './data/work:/opt/graphdb/work'
+            - './data/logs:/opt/graphdb/logs'
+        restart: always
+
+networks:
+    default:
+        external:
+          name: distillery

distillery/resources/compose/triplestore/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Because we want to run graphdb as a limited user
+# we need to make sure that the volumes are writable. 
+# Because of that, we 'chown'
+
+chown graphdb:graphdb /opt/graphdb/data
+chown graphdb:graphdb /opt/graphdb/work
+chown graphdb:graphdb /opt/graphdb/logs
+
+# switch to the graphdb user, and run graphdb
+su graphdb -c "/opt/graphdb/bin/graphdb $@"

distillery/resources/compose/web/docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3.7"
+
+services:
+  nginx-proxy:
+    image: nginxproxy/nginx-proxy:alpine
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "vhost:/etc/nginx/vhost.d"
+      - "htpasswd:/etc/nginx/htpasswd"
+      - "html:/usr/share/nginx/html"
+      - "/var/run/docker.sock:/tmp/docker.sock:ro"
+      - "certs:/etc/nginx/certs"
+    labels:
+      com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: true
+    restart: always
+    networks:
+      - default
+
+  letsencrypt-nginx-proxy-companion:
+    image: jrcs/letsencrypt-nginx-proxy-companion
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "htpasswd:/etc/nginx/htpasswd"
+      - "vhost:/etc/nginx/vhost.d"
+      - "html:/usr/share/nginx/html"
+      - "/var/run/docker.sock:/tmp/docker.sock:ro"
+      - "certs:/etc/nginx/certs"
+    restart: always
+    networks:
+      - default
+    depends_on:
+      - nginx-proxy
+
+volumes:
+  vhost:
+  html:
+  certs:
+  htpasswd:
+
+networks:
+  default:
+    external:
+      name: distillery

distillery/resources/templates/bookkeeping/create.sql

@@ -0,0 +1,28 @@
+-- create the database if it doesn't exist yet
+CREATE DATABASE IF NOT EXISTS `${DATABASE}`;
+
+-- use the database we just created
+USE `${DATABASE}`;
+
+-- create the bookkeeping table
+CREATE TABLE IF NOT EXISTS `${TABLE}`(
+    -- automatically created fields
+    pk INT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+    created DATETIME DEFAULT CURRENT_TIMESTAMP,
+    
+    -- slug of the website
+    slug TEXT NOT NULL UNIQUE,
+
+    -- local file path
+    filesystem_base TEXT NOT NULL,
+    
+    -- sql access credentials
+    sql_database TEXT NOT NULL,
+    sql_user TEXT NOT NULL,
+    sql_password TEXT NOT NULL,
+
+    -- graphdb credentials
+    graphdb_repository TEXT NOT NULL,
+    graphdb_user TEXT NOT NULL,
+    graphdb_password TEXT NOT NULL
+);

distillery/resources/templates/repository/graphdb-repo.ttl

@@ -0,0 +1,57 @@
+# This file is used to initialize a new GraphDB repository. 
+# In this file the variables ${GRAPHDB_REPO} and ${INSTANCE_DOMAIN} will be replaced. 
+# All other variables will be left untouched. 
+
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix rep: <http://www.openrdf.org/config/repository#>.
+@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
+@prefix sail: <http://www.openrdf.org/config/sail#>.
+@prefix owlim: <http://www.ontotext.com/trree/owlim#>.
+
+[] a rep:Repository ;
+    rep:repositoryID "${GRAPHDB_REPO}" ;
+    rdfs:label "${INSTANCE_DOMAIN}" ;
+    rep:repositoryImpl [
+        rep:repositoryType "graphdb:FreeSailRepository" ;
+        sr:sailImpl [
+            sail:sailType "graphdb:FreeSail" ;
+
+            owlim:owlim-license "" ;
+
+            owlim:base-URL "http://${INSTANCE_DOMAIN}/#" ;
+            owlim:defaultNS "" ;
+            owlim:entity-index-size "10000000" ;
+            owlim:entity-id-size  "32" ;
+            owlim:imports "" ;
+            owlim:repository-type "file-repository" ;
+            owlim:ruleset "empty" ;
+            owlim:storage-folder "storage" ;
+
+            owlim:enable-context-index "false" ;
+            owlim:cache-memory "80m" ;
+            owlim:tuple-index-memory "80m" ;
+
+            owlim:enablePredicateList "false" ;
+            owlim:predicate-memory "0%" ;
+
+            owlim:fts-memory "0%" ;
+            owlim:ftsIndexPolicy "never" ;
+            owlim:ftsLiteralsOnly "true" ;
+
+            owlim:in-memory-literal-properties "false" ;
+            owlim:enable-literal-index "true" ;
+            owlim:index-compression-ratio "-1" ;
+
+            owlim:check-for-inconsistencies "false" ;
+            owlim:disable-sameAs  "false" ;
+            owlim:enable-optimization  "true" ;
+            owlim:transaction-mode "safe" ;
+            owlim:transaction-isolation "true" ;
+            owlim:query-timeout  "0" ;
+            owlim:query-limit-results  "0" ;
+            owlim:throw-QueryEvaluationException-on-timeout "false" ;
+            owlim:useShutdownHooks  "true" ;
+            owlim:read-only "false" ;
+            owlim:nonInterpretablePredicates "http://www.w3.org/2000/01/rdf-schema#label;http://www.w3.org/1999/02/22-rdf-syntax-ns#type;http://www.ontotext.com/owlim/ces#gazetteerConfig;http://www.ontotext.com/owlim/ces#metadataConfig" ;
+        ]
+    ].

distillery/resources/templates/repository/graphdb-user.json

@@ -0,0 +1,8 @@
+{
+    "username": "${GRAPHDB_USER}",
+    "grantedAuthorities": [
+        "ROLE_USER",
+        "READ_REPO_${GRAPHDB_REPO}",
+        "WRITE_REPO_${GRAPHDB_REPO}"
+    ]
+}

distillery/resources/templates/runtime-config/environment

@@ -0,0 +1,17 @@
+#######################
+# Meta Settings
+#######################
+
+# Real path for volumes to be stored
+REAL_PATH=${REAL_PATH}
+
+#######################
+### Web Server settings
+#######################
+# the hostname for the website
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+# optional letsencrypt support
+# when blank, ignore
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}