Refactor Package structure

This commit cleans up the package structure, to make two new top-level
packages `internal` (for internal-use packages) and `pkg` (for general
shared utility code).

internal/env/instances/barrel/.dockerignore

@@ -0,0 +1,8 @@
+# Ignore everything
+*
+
+# allow the following files:
+!conf/*
+!scripts/*
+!patch/*
+!wisskiutils/*

internal/env/instances/barrel/.env.sample

@@ -0,0 +1,28 @@
+#######################
+# Meta Settings
+#######################
+
+# Real path for volumes to be stored
+DATA_PATH=/var/www/deploy/instances/example.slug/data
+UTILS_DIR=/var/www/deploy/runtime/utils/
+
+#######################
+### Web Server settings
+#######################
+# the hostname for the website
+VIRTUAL_HOST=example.com
+
+# optional letsencrypt support
+# when blank, ignore
+LETSENCRYPT_HOST=
+LETSENCRYPT_EMAIL=
+
+### SQL settings
+MYSQL_HOST=mysql
+MYSQL_USER=user
+MYSQL_PASS=pass
+
+### GraphDB settings
+GRAPHDB_HOST=graphdb
+GRAPHDB_USER=user
+GRAPHDB_PASS=pass

internal/env/instances/barrel/Dockerfile

@@ -0,0 +1,106 @@
+FROM docker.io/library/php:8.0-apache-bullseye
+ARG COMPOSER_VERSION=2.3.8
+WORKDIR /var/www
+
+# install and enable the various required php extension
+RUN apt-get update && apt-get install -y \
+        curl \
+        default-mysql-client \
+        git \
+        imagemagick \
+        libcurl4-openssl-dev \
+        libfreetype6-dev \
+        libicu-dev \
+        libjpeg62-turbo-dev \
+        libpng-dev \
+        libssh2-1-dev \
+        libwebp-dev \
+        libxml2-dev \
+        libxpm-dev \
+        sudo \
+        unzip \
+        vim \
+        zip \
+    && \
+    docker-php-source extract && \
+    mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" && \
+    pear config-set php_ini "$PHP_INI_DIR/php.ini" && \
+    docker-php-ext-configure gd \
+        --enable-gd \
+        --with-webp \
+        --with-jpeg \
+        --with-xpm \
+        --with-freetype \
+        --enable-gd-jis-conv \
+    && \
+    docker-php-ext-install \
+        curl \
+        gd \
+        intl \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        soap \
+        xml \
+    && \
+    pecl install xmlrpc-1.0.0RC3 && \
+    pecl install ssh2-1.3.1 && \
+    pecl install apcu-5.1.21 && \
+    pecl install uploadprogress-2.0.2 && \
+    docker-php-ext-enable \
+        apcu \
+        curl \
+        gd \
+        intl \
+        mysqli \
+        mysqli \
+        opcache \
+        pdo_mysql \
+        soap \
+        ssh2 \
+        uploadprogress \
+        xml \
+        xmlrpc \
+    && \
+    docker-php-source delete
+
+# enable the apache rewrite mod
+RUN a2enmod rewrite
+
+# install composer and add it to path
+RUN curl -sS https://getcomposer.org/installer | php -- --version=$COMPOSER_VERSION && \
+    mv composer.phar /usr/local/bin/composer
+ENV PATH "/usr/local/bin:/var/www/data/project/vendor/bin:$PATH"
+
+# remove default configuration
+RUN rm /etc/apache2/sites-available/*.conf && \
+    rm /etc/apache2/sites-enabled/*.conf
+
+ADD patch/easyrdf.patch /patch/easyrdf.patch
+ADD patch/triples.patch /patch/triples.patch
+
+# Add wisski configuration
+ADD conf/ports.conf /etc/apache2/ports.conf
+ADD conf/wisski.conf /etc/apache2/sites-available/wisski.conf
+ADD conf/wisski.ini /usr/local/etc/php/conf.d/wisski.ini
+RUN a2ensite wisski
+
+# volumes for composer
+VOLUME /var/www/.composer
+VOLUME /var/www/data
+
+# Add and configure the entrypoint
+ADD scripts/entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT [ "/bin/bash", "/entrypoint.sh" ]
+CMD ["apache2-foreground"]
+
+# Add the provision script and WissKI utils
+ADD scripts/provision_container.sh /provision_container.sh
+ADD wisskiutils/ /wisskiutils
+
+# Add the user_shell.sh
+ADD scripts/user_shell.sh /user_shell.sh
+
+# expose port 8080
+EXPOSE 8080

internal/env/instances/barrel/conf/ports.conf

@@ -0,0 +1,4 @@
+# This file configures where apache should listen. 
+# Because we are running as a limited user, we want to listen on a high port. 
+# For this we use port 8080
+Listen 8080

internal/env/instances/barrel/conf/wisski.conf

@@ -0,0 +1,24 @@
+<VirtualHost *:8080>
+    # the document root -- /var/www/data/project/web
+    DocumentRoot /var/www/data/project/web
+
+    <Directory /var/www/data/project/web>
+        # add types for .owl and .rdf
+        AddType application/rdf+xml .owl
+        AddType application/rdf+xml .rdf
+
+        # Rewrite the 'ontology' directory
+        RewriteEngine On
+        RewriteOptions InheritDownBefore
+        ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,END]
+        ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [END]
+
+        # Allow overrides of symlinks
+        Options Indexes FollowSymLinks
+        AllowOverride All
+        Require all granted
+    </Directory>
+
+    ErrorLog /dev/stderr
+    CustomLog /dev/stdout combined
+</VirtualHost>

internal/env/instances/barrel/conf/wisski.ini

@@ -0,0 +1,14 @@
+; File Uploads up to 1GB
+file_uploads = On
+upload_max_filesize = 1000M
+post_max_size = 1000M
+
+; Composer uses an absurd amount of memory
+; 4GB ought to be enough
+memory_limit = 4G
+
+; Increase various limits for some long running WissKI operations
+max_execution_time = 3000
+max_input_time = 600
+max_input_nesting_level = 640
+max_input_vars = 10000

internal/env/instances/barrel/docker-compose.yml

@@ -0,0 +1,33 @@
+version: "3.7"
+
+services:
+  barrel:
+    build: .
+    restart: always
+    hostname: ${VIRTUAL_HOST}.wisski
+    environment:
+      # port and hostname for this image to use
+      VIRTUAL_HOST: ${VIRTUAL_HOST}
+      VIRTUAL_PORT: 8080
+      
+      # optional letsencrypt email
+      LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
+      LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
+    
+    # label it with the current slug
+    labels:
+      eu.wiss-ki.barrel.slug: ${SLUG}
+      eu.wiss-ki.barrel.authfile: /var/www/.ssh/authorized_keys,/var/www/.ssh/global_authorized_keys
+    
+    # volumes that are mounted
+    volumes:
+      - ${GLOBAL_AUTHORIZED_KEYS_FILE}:/var/www/.ssh/global_authorized_keys:ro
+      - ${DATA_PATH}/.composer:/var/www/.composer
+      - ${DATA_PATH}/data:/var/www/data
+      - ${DATA_PATH}/authorized_keys:/var/www/.ssh/authorized_keys
+      - ${RUNTIME_DIR}:/runtime:ro
+
+networks:
+  default:
+    name: distillery
+    external: true

internal/env/instances/barrel/patch/easyrdf.patch

@@ -0,0 +1,4 @@
+281c281
+<         if (preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]+)|", $status, $m)) {
+---
+>         if(preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]*)|", $status, $m)) {

internal/env/instances/barrel/patch/triples.patch

@@ -0,0 +1,8 @@
+100c100
+<             if($result->o instanceof \EasyRdf_Resource) {
+---
+> 	    if($result->o instanceof \EasyRdf\Resource) {
+118c118
+<               $object_text = $result->o->getValue();
+---
+>               $object_text = $result->o->dumpValue('string');

internal/env/instances/barrel/scripts/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This script contains 
+
+# chown the volumes to make sure they can be read and written by the limited user
+chown www-data:www-data /var/www
+chown www-data:www-data /var/www/.composer
+chown www-data:www-data /var/www/data/
+
+# run the original entrypoint
+docker-php-entrypoint "$@"

internal/env/instances/barrel/scripts/provision_container.sh

@@ -0,0 +1,174 @@
+#!/bin/bash
+set -e
+
+function log_info() {
+   echo -e "\033[1m$1\033[0m"
+}
+
+function log_ok() {
+   echo -e "\033[0;32m$1\033[0m"
+}
+
+log_info " => Reading configuration variables"
+
+INSTANCE_DOMAIN="$1"
+echo "INSTANCE_DOMAIN=$INSTANCE_DOMAIN"
+shift 1
+
+MYSQL_DATABASE="$1"
+echo "MYSQL_DATABASE=$MYSQL_DATABASE"
+MYSQL_USER="$2"
+echo "MYSQL_USER=$MYSQL_USER"
+MYSQL_PASSWORD="$3"
+echo "MYSQL_PASSWORD=$MYSQL_PASSWORD"
+
+shift 3
+
+GRAPHDB_REPO="$1"
+echo "GRAPHDB_REPO=$GRAPHDB_REPO"
+GRAPHDB_USER="$2"
+echo "GRAPHDB_USER=$GRAPHDB_USER"
+GRAPHDB_PASSWORD="$3"
+echo "GRAPHDB_PASSWORD=$GRAPHDB_PASSWORD"
+shift 3
+
+GRAPHDB_HEADER="$(printf "%s:%s" "$GRAPHDB_USER" "$GRAPHDB_PASSWORD" | base64 -w 0)"
+
+DRUPAL_USER="$1"
+echo "DRUPAL_USER=$DRUPAL_USER"
+DRUPAL_PASS="$2"
+echo "DRUPAL_PASS=$DRUPAL_PASS"
+shift 2
+
+DRUPAL_VERSION="$1"
+echo "DRUPAL_VERSION=$DRUPAL_VERSION"
+shift 1
+
+WISSKI_VERSION="$1"
+echo "WISSKI_VERSION=$WISSKI_VERSION"
+shift 1
+
+log_info " => Preparing installation environment"
+BASE_DIR="/var/www/data"
+COMPOSER_DIR="$BASE_DIR/project"
+WEB_DIR="$COMPOSER_DIR/web"
+ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
+
+log_info " => Creating '$COMPOSER_DIR'"
+mkdir -p "$COMPOSER_DIR"
+cd "$COMPOSER_DIR"
+
+# workaround for making the drupal sites directory writable
+function drupal_sites_permission_workaround() {
+    chmod -R u+w "$WEB_DIR/sites/" || true
+}
+
+# install a module with composer and enable it with drush
+# Example:
+#
+# composer_install_and_enable << EOF
+# drupal/some_module:1.23 some_module
+# drupal/other_module:2.34
+# EOF
+# 
+# Will install both modules, but only enable the first one.
+function composer_install_and_enable() {
+    while IFS= read -r line; do
+        echo "$line" | (
+            read composer drush;
+            drupal_sites_permission_workaround
+            composer require "$composer"
+            if [ -n "$drush" ]; then
+                drush pm-enable --yes "$drush"
+            fi
+        )
+    done
+}
+
+
+# Create a new composer project. 
+log_info " => Creating composer project"
+if [ -z "${DRUPAL_VERSION}" ]; then
+    composer --no-interaction create-project 'drupal/recommended-project:^9.0.0' .
+else
+    composer --no-interaction create-project "drupal/recommended-project:$DRUPAL_VERSION" .
+fi
+
+# needed for composer > 2.2
+composer --no-interaction config allow-plugins true
+
+# Install drush so that we can automate a lot of things
+log_info " => Installing 'drush'"
+composer require drush/drush
+
+# Use 'drush' to run the site-installation. 
+# Here we need to use the username, password and database creds we made above. 
+log_info " => Running drupal installation scripts"
+drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
+    --account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
+    --db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@sql/${MYSQL_DATABASE}
+drupal_sites_permission_workaround
+
+# create a directory for ontologies. 
+log_info " => Creating '$ONTOLOGY_DIR'"
+mkdir -p "$ONTOLOGY_DIR"
+
+# Install the Wisski packages. 
+log_info " => Installing Wisski packages"
+cd "$COMPOSER_DIR"
+
+# install the development version when requested
+if [ -z "${WISSKI_VERSION}" ]; then
+    composer require 'drupal/wisski'
+else
+    composer require "drupal/wisski:$WISSKI_VERSION"
+fi
+
+# Install dependencies of WissKI
+log_info " => Installing and patching Wisski dependencies"
+pushd "$WEB_DIR/modules/contrib/wisski"
+composer install
+
+# Patch EasyRDF (for now)
+EASYRDF_RESPONSE="./vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
+if [ -f "$EASYRDF_RESPONSE" ]; then
+    patch -N "$EASYRDF_RESPONSE" < "/patch/easyrdf.patch"
+fi
+popd
+
+log_info " => Installing and enabling additional modules"
+composer_install_and_enable << EOF
+drupal/inline_entity_form:^1.0@RC
+drupal/imagemagick
+drupal/image_effects
+drupal/colorbox
+drupal/devel:^4.1 devel
+drupal/geofield:^1.40 geofield
+drupal/geofield_map:^2.85 geofield_map
+drupal/imce:^2.4 imce
+EOF
+
+log_info " => Enable Wisski modules"
+drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
+drupal_sites_permission_workaround
+
+log_info " => Setting up WissKI Salz Adapter"
+drush php:script /wisskiutils/create_adapter.php "$INSTANCE_DOMAIN" "$GRAPHDB_REPO" "$GRAPHDB_HEADER"
+
+log_info " => Updating TRUSTED_HOST_PATTERNS in settings.php"
+
+/bin/bash /wisskiutils/set_trusted_host.sh
+
+log_info " => Running initial cron"
+drush core-cron
+
+log_info " => Provisioning is now complete. "
+log_ok "Your installation details are as follows:"
+function printdetails() {
+    echo "URL:                  http://$INSTANCE_DOMAIN"
+    echo "Username:             $DRUPAL_USER"
+    echo "Password:             $DRUPAL_PASS"
+}
+printdetails
+
+exit 0

internal/env/instances/barrel/scripts/user_shell.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# This script is used to start a user shell inside the docker container. 
+cd "/var/www/data/project"
+sudo -u www-data "PATH=/var/www/data/project/vendor/bin:$PATH" /bin/bash "$@"

internal/env/instances/barrel/wisskiutils/create_adapter.php

@@ -0,0 +1,61 @@
+<?php
+
+/**
+ * This script will automatically create a WissKI Salz Adapter for use within the distillery.
+ * It will not update any existing adapter and is rather primitive.
+ */
+
+$argc = $_SERVER['argc']-3;
+$argv = array_slice($_SERVER['argv'], 3);
+
+// read parameters from the command line
+if ($argc != 3) {
+    die("Usage: drush php:script create_adapter.php INSTANCE_DOMAIN GRAPHDB_REPO HEADER");
+}
+$INSTANCE_DOMAIN = $argv[0];
+$GRAPHDB_REPO = $argv[1];
+$HEADER = $argv[2];
+
+//
+// PROPERTIES FOR THE ADAPTER
+//
+
+$id = 'default'; // id
+$type = 'sparql11_with_pb'; // plugin
+$machine_name = 'default'; // machine-name
+$label = 'Default WissKI Distillery Adapter';
+$description = 'Adapter for ' . $INSTANCE_DOMAIN; // description
+$writable = TRUE; // writable
+$is_preferred_local_store = TRUE; // is_preferred_local_store
+$header = $HEADER; // header
+$read_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO; // read_url
+$write_url = 'http://triplestore:7200/repositories/' . $GRAPHDB_REPO . '/statements'; // write_url
+$is_federatable = TRUE; // is_federatable
+$default_graph_uri = 'https://' . $INSTANCE_DOMAIN . '/';
+$same_as_properties = ['http://www.w3.org/2002/07/owl#sameAs']; // same_as_properties
+$ontology_graphs = []; // ontology_graphs
+
+//
+// Do the creation!
+//
+
+$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+$adapter = $storage->create([
+    "id" => $id,
+    "label" => $label,
+    "description" => $description,
+]);
+$adapter->setEngineConfig([
+    "id" => $type,
+    "machine-name" => $machine_name,
+    "header" => $header,
+    "writeable" => $writable,
+    "is_preferred_local_store" => $is_preferred_local_store,
+    "read_url" => $read_url,
+    "write_url" => $write_url,
+    "is_federatable" => $is_federatable,
+    "default_graph" => $default_graph_uri,
+    "same_as_properties" => $same_as_properties,
+    "ontology_graphs" => $ontology_graphs,
+]);
+$adapter->save();

internal/env/instances/barrel/wisskiutils/export_pathbuilder.php

@@ -0,0 +1,63 @@
+<?php
+
+/**
+ * This script will list all the URIs that this system is aware of.
+ * This works by listing all the default graph uris of all the adapters.
+ */
+
+use Drupal\wisski_pathbuilder\Entity\WisskiPathEntity;
+
+// load all the pathbuilders
+$pbs = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->loadMultiple();
+
+// map over the pathbuilders
+$xmls = array_map(function($pb) {
+    $xml = new \SimpleXMLElement("<pathbuilderinterface></pathbuilderinterface>");
+
+    $paths = $pb->getAllPaths();
+    foreach ($paths as $key => $path) {
+        $id = $path->getID();
+
+        $path = $pb->getPbPath($id);
+
+        $pathChild = $xml->addChild("path");
+        $pathObject = WisskiPathEntity::load($id);
+
+        foreach ($path as $subkey => $value) {
+
+            if (in_array($subkey, ['relativepath'])) {
+                continue;
+            }
+
+            if ($subkey == "parent") {
+                $subkey = "group_id";
+            }
+
+            $pathChild->addChild($subkey, htmlspecialchars($value));
+        }
+
+        $pathArray = $pathChild->addChild('path_array');
+        foreach ($pathObject->getPathArray() as $subkey => $value) {
+            $pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value);
+        }
+
+        $pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty()));
+        $pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName()));
+        $pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb()));
+        $pathChild->addChild('description', htmlspecialchars($pathObject->getDescription()));
+        $pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid()));
+        if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") {
+            $pathChild->addChild('is_group', "1");
+        } else {
+            $pathChild->addChild('is_group', "0");
+        }
+        $pathChild->addChild('name', htmlspecialchars($pathObject->getName()));
+    }
+
+    // turn it into XML
+    $dom = dom_import_simplexml($xml)->ownerDocument;
+    $dom->formatOutput = TRUE;
+    return $dom->saveXML();
+}, $pbs);
+
+echo json_encode($xmls);

internal/env/instances/barrel/wisskiutils/list_uri_prefixes.php

@@ -0,0 +1,19 @@
+<?php
+
+/**
+ * This script will list all the URIs that this system is aware of.
+ * This works by listing all the default graph uris of all the adapters.
+ */
+
+// iterate over all adapters
+$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+foreach ($storage->loadMultiple() as $adapter) {
+    // read the configuration, and check if we have a default graph
+    $conf = $adapter->getEngine()->getConfiguration();
+    if(!array_key_exists('default_graph', $conf)) {
+        continue;
+    }
+
+    // and echo it out
+    echo $conf['default_graph'] . "\n";
+}

internal/env/instances/barrel/wisskiutils/set_trusted_host.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This utility script can be used to configure the trusted host settings inside of settings.php. 
+# It doesn't take care of corner cases and should only be used when needed. 
+
+INSTANCE_DOMAIN="$(hostname -f)"
+INSTANCE_DOMAIN="${INSTANCE_DOMAIN%.wisski}"
+
+TRUSTED_HOST_PATTERN="${INSTANCE_DOMAIN//\./\\\\.}"
+TRUSTED_HOST_PATTERNS='["'$TRUSTED_HOST_PATTERN'"]'
+
+echo "Setting 'trusted_host_patterns' to $TRUSTED_HOST_PATTERNS"
+bash /wisskiutils/settings_php_set.sh 'trusted_host_patterns' "$TRUSTED_HOST_PATTERNS"

internal/env/instances/barrel/wisskiutils/settings_php_get.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# settings_php_get.sh name
+# Gets the 'settings_php_get.php' setting 'name' as json-encoded value, or null when it does not exist.
+
+NAME=$1
+
+if [ -z "$NAME" ]; then
+    echo "Usage: get_settings_setting.sh NAME"
+    exit 1
+fi;
+
+echo "$NAME" | drush php:eval '
+  use \Drupal\Core\Site\Settings;
+  $name=trim(file_get_contents("php://stdin"));
+  echo json_encode(Settings::get($name));
+';

internal/env/instances/barrel/wisskiutils/settings_php_set.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# settings_php_set.sh name value
+# Sets the 'settings.php' setting 'name' to 'value'.
+# Value must be json-encoded. 
+
+NAME=$1
+VALUE=$2
+
+if [ -z "$NAME" ]; then
+    echo "Usage: settings_php_set.sh NAME VALUE"
+    exit 1
+fi;
+
+if [ -z "$VALUE" ]; then
+    echo "Usage: settings_php_set.sh NAME VALUE"
+    exit 1
+fi;
+
+cd /var/www/data/project
+chmod u+w web/sites/default/settings.php
+
+(echo "$NAME"; echo "$VALUE" ) | drush php:eval '
+    include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
+
+    // read NAME and VALUE from STDIN
+    $content=file_get_contents("php://stdin");  
+    $newline=strpos($content, "\n");
+    $name=trim(substr($content, 0, $newline));
+    $jvalue=trim(substr($content, $newline + 1));
+
+    // decode json values
+    $value = @json_decode($jvalue);
+    if ($value === null && json_last_error() !== JSON_ERROR_NONE) {
+        echo "Invalid JSON, cannot update settings.php. \n";
+        return 1;
+    }
+
+    // make parameters to drush_rewrite_settings
+    $settings["settings"][$name] = (object)[
+        "value" => $value,
+        "required" => TRUE,
+    ];
+
+    // find the actual settings.php file to rewrite
+    $filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
+    drupal_rewrite_settings($settings, $filename);
+
+    echo "Wrote " . $filename . "\n";
+    return 0;   
+';
+EXIT=$?
+
+chmod u-w web/sites/default/settings.php
+
+exit $?