rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor templating and drupal 9 support

Browse source
This commit is contained in:
Tom Wiesing 2020-06-10 15:43:00 +02:00
parent eabf4a2ec4
commit f22e658183
No known key found for this signature in database
GPG key ID: DC1F29F2BC78AB15
16 changed files with 275 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

6
factory/.env.example
View file

@ -1,4 +1,8 @@
# All WissKi and Drupal Installations are contained within a single directory.
# Sometimes when http(s) is enabled, it is advisable to listen on a port different from 80.
# The public port can be configured here.
PUBLIC_PORT=80
# All WissKI and Drupal Installations are contained within a single directory.
# The name of each subfolder corresponds to the appropriate domain name.
# This variable determines the subfolder to place installations into.
DRUPAL_ROOT=/var/www/factory

7
factory/lib/10_config.sh
View file

@ -125,4 +125,11 @@ if ! is_valid_number "$PASSWORD_LENGTH"; then
exit 1;
fi
# The 'PUBLIC_PORT' must be a valid number.
if ! is_valid_number "$PUBLIC_PORT"; then
log_error "Variable 'PUBLIC_PORT' is missing or not a valid number. ";
log_info "Please verify that it is set correctly in '.env'. ";
exit 1;
fi
log_ok "Read and validated configuration file. "

1
factory/lib/20_slug.sh
View file

@ -62,6 +62,7 @@ GRAPHDB_REPO="${GRAPHDB_REPO_PREFIX}${USERNAME_BASE}"
BASE_DIR="$DRUPAL_ROOT/$INSTANCE_DOMAIN"
COMPOSER_DIR="$BASE_DIR/project"
WEB_DIR="$COMPOSER_DIR/web"
ONTOLOGY_DIR="$WEB_DIR/sites/default/files/ontology"
# Setup aliases for drush and composer.
alias composer="sudo -u $SYSTEM_USER /usr/local/bin/composer"

34
factory/lib/30_templates.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
set -e
# This is a library file.
# It should be 'source'd only, if it is not we bail out here.
if [[ "$0" = "$BASH_SOURCE" ]]; then
echo "This file should not be executed directly, it should be 'source'd only. "
exit 1;
fi
TEMPLATE_DIR="$SCRIPT_DIR/resources/templates/"
# load_template will load a template $1 from the template directory
# and replace ${$2} with $3, ${$4} with $5 etc.
# echoes out the replaced template
function load_template() {
# read the template then remove the argument
TEMPLATE=`cat "$TEMPLATE_DIR/$1"`
shift 1;
# while we have a variable to substiute
while [ ! -z "$1" ]
do
# do the substitution
TEMPLATE="${TEMPLATE//\$\{$1\}/$2}"
shift 2
done;
# finally echo out the template
echo "$TEMPLATE"
}
# path where common apache files will be installed.
WISSKI_COMMON_PATH="/etc/apache2/conf/wisski"

0
factory/lib/30_utils.sh → factory/lib/40_utils.sh
View file

3
factory/lib/lib.sh
View file

@ -17,4 +17,5 @@ LIB_DIR="$SCRIPT_DIR/lib"
source "$LIB_DIR/00_init.sh";
source "$LIB_DIR/10_config.sh";
source "$LIB_DIR/20_slug.sh";
source "$LIB_DIR/30_utils.sh";
source "$LIB_DIR/30_templates.sh";
source "$LIB_DIR/40_utils.sh";

183
factory/provision.sh
View file

@ -3,7 +3,7 @@
# This script will provision a new Drupal instance and make it available to apache.
# Usage: sudo ./provision.sh $SLUG
# In case the installation fails, it will bail out and leave you with an incomplete installation.
# To delete an incomplete installation, use the ./remove.sh script, or try fixing the error manually.
# To delete an incomplete installation, use the ./purge.sh script, or try fixing the error manually.
set -e
# read the lib/shared.sh and read the slug argument.
@ -13,6 +13,24 @@ source "$DIR/lib/lib.sh"
require_slug_argument
# A global flag 'USE_DRUPAL_9' can be set to enable drupal 9 support.
# We print out the value of the flag here
if [ -z "${USE_DRUPAL_9}" ]; then
log_info " => Will install stable Drupal 8 (Use 'USE_DRUPAL_9=1' for Drupal 9). "
else
log_info " => Will install experimental Drupal 9 version ('USE_DRUPAL_9' was set)"
fi
log_info " => Validing configuration"
# If the base directory already exists, we might have accidentally picked a name that already exists.
# In that case we bail out for safety reasons.
if [ -d "$BASE_DIR" ]; then
echo "'$BASE_DIR' already exists. "
echo "Aborting provisioning, please make sure you picked a unique name. "
exit 1
fi
# Check that the apache2 config is correct.
# This is a sanity test so that we don't randomly fail later because of bad config.
log_info " => Checking apache configuration"
@ -29,27 +47,23 @@ sudo -u "$SYSTEM_USER" mkdir -p "$COMPOSER_DIR"
cd "$COMPOSER_DIR"
# Write out a new apache configuration file into /etc/apache2/sites-available.
# We will need to substiute in some configuration directories.
# We will need to substitute in some configuration directories.
log_info " => Writing new apache configuration file"
cat << EOF >> "$APACHE_CONFIG_SITE_AVAILABLE"
<VirtualHost *:80>
DocumentRoot $WEB_DIR
ServerName $INSTANCE_DOMAIN
AssignUserId $SYSTEM_USER $SYSTEM_USER
<Directory $WEB_DIR>
Options Indexes FollowSymLinks
AllowOverride All
Require all granted
</Directory>
ErrorLog \${APACHE_LOG_DIR}/error.log
CustomLog \${APACHE_LOG_DIR}/access.log combined
</VirtualHost>
EOF
load_template "wisski-site.conf" \
"PUBLIC_PORT" "${PUBLIC_PORT}" \
"WEB_DIR" "${WEB_DIR}" \
"INSTANCE_DOMAIN" "${INSTANCE_DOMAIN}" \
"SYSTEM_USER" "${SYSTEM_USER}" \
"WISSKI_COMMON_PATH" "${WISSKI_COMMON_PATH}" \
> "${APACHE_CONFIG_SITE_AVAILABLE}"
# Create a new composer project.
log_info " => Creating composer project"
composer create-project drupal/recommended-project .
if [ -z "${USE_DRUPAL_9}" ]; then
composer create-project 'drupal/recommended-project:^8.9.0' .
else
composer create-project 'drupal/recommended-project:^9.0.0' .
fi
composer require drush/drush
# Randomly generate the database name and user we will configure.
@ -73,90 +87,38 @@ DRUPAL_PASS="$(randompw)"
# Use 'drush' to run the site-installation.
# Here we need to use the username, password and database creds we made above.
log_info " => Running drupal installation scripts"
drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} --account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS --db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@localhost/${MYSQL_DATABASE}
drush site-install standard --yes --site-name=${INSTANCE_DOMAIN} \
--account-name=$DRUPAL_USER --account-pass=$DRUPAL_PASS \
--db-url=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@localhost/${MYSQL_DATABASE}
drupal_sites_permission_workaround
# Create a new repository for GraphDB.
# First write out the configuration into a new directory.
log_info " => Writing GraphDB configuration in temporary directory"
tmpdir="$(mktemp -d)"
cd "$tmpdir"
cat << EOF > repo-config.ttl
# Creates a new GraphDB repository with Wisski
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
@prefix rep: <http://www.openrdf.org/config/repository#>.
@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
@prefix sail: <http://www.openrdf.org/config/sail#>.
@prefix owlim: <http://www.ontotext.com/trree/owlim#>.
[] a rep:Repository ;
rep:repositoryID "$GRAPHDB_REPO" ;
rdfs:label "$INSTANCE_DOMAIN" ;
rep:repositoryImpl [
rep:repositoryType "graphdb:FreeSailRepository" ;
sr:sailImpl [
sail:sailType "graphdb:FreeSail" ;
owlim:owlim-license "" ;
owlim:base-URL "http://$INSTANCE_DOMAIN/owlim#" ;
owlim:defaultNS "" ;
owlim:entity-index-size "10000000" ;
owlim:entity-id-size "32" ;
owlim:imports "" ;
owlim:repository-type "file-repository" ;
owlim:ruleset "empty" ;
owlim:storage-folder "storage" ;
owlim:enable-context-index "false" ;
owlim:cache-memory "80m" ;
owlim:tuple-index-memory "80m" ;
owlim:enablePredicateList "false" ;
owlim:predicate-memory "0%" ;
owlim:fts-memory "0%" ;
owlim:ftsIndexPolicy "never" ;
owlim:ftsLiteralsOnly "true" ;
owlim:in-memory-literal-properties "false" ;
owlim:enable-literal-index "true" ;
owlim:index-compression-ratio "-1" ;
owlim:check-for-inconsistencies "false" ;
owlim:disable-sameAs "false" ;
owlim:enable-optimization "true" ;
owlim:transaction-mode "safe" ;
owlim:transaction-isolation "true" ;
owlim:query-timeout "0" ;
owlim:query-limit-results "0" ;
owlim:throw-QueryEvaluationException-on-timeout "false" ;
owlim:useShutdownHooks "true" ;
owlim:read-only "false" ;
owlim:nonInterpretablePredicates "http://www.w3.org/2000/01/rdf-schema#label;http://www.w3.org/1999/02/22-rdf-syntax-ns#type;http://www.ontotext.com/owlim/ces#gazetteerConfig;http://www.ontotext.com/owlim/ces#metadataConfig" ;
]
].
EOF
# Create the configuration and use the configuration generated above.
# Use the template for this.
# TODO: Permissions for GraphdDB
log_info "Generating new GraphDB repository '$GRAPHDB_REPO'"
curl -X POST\
http://127.0.0.1:7200/rest/repositories\
-H 'Content-Type: multipart/form-data'\
-F "config=@repo-config.ttl"
log_info " => Generating new GraphDB repository '$GRAPHDB_REPO'"
load_template "graphdb-repo.ttl" "GRAPHDB_REPO" "${GRAPHDB_REPO}" "INSTANCE_DOMAIN" "${INSTANCE_DOMAIN}" | \
curl -X POST \
http://127.0.0.1:7200/rest/repositories \
-H 'Content-Type: multipart/form-data' \
-F "config=@-"
# Remove the temporary directory.
cd ..
rm -rf "$tmpdir"
# create a directory for ontologies.
log_info " => Creating '$ONTOLOGY_DIR'"
mkdir -p "$ONTOLOGY_DIR"
# Install the Wisski packages.
log_info " => Installing Wisski packages"
cd "$COMPOSER_DIR"
drupal_sites_permission_workaround
composer require drupal/wisski
# install the development version when requested
if [ -z "${USE_DRUPAL_9}" ]; then
composer require 'drupal/wisski'
else
composer require 'drupal/wisski:2.x-dev'
fi
drupal_sites_permission_workaround
composer require drupal/inline_entity_form
@ -170,13 +132,35 @@ composer require drupal/image_effects
drupal_sites_permission_workaround
composer require drupal/colorbox
log_info " => Installation is now technically complete. "
log_ok "Some things below may fail. If that is the case, run: "
log_ok "$ a2ensite \"${INSTANCE_DOMAIN}\""
log_ok "$ systemctl reload apache2"
log_ok "$ $SCRIPT_DIR/shell.sh $SLUG"
log_ok "Your installation details are as follows"
function printdetails() {
echo "URL: http://$INSTANCE_DOMAIN"
echo "Username: $DRUPAL_USER"
echo "Password: $DRUPAL_PASS"
log_info " => Your GraphDB details (for WissKI Salz) are: "
echo "Read URL: http://127.0.0.1:7200/repositories/$GRAPHDB_REPO"
echo "Write URL: http://127.0.0.1:7200/repositories/$GRAPHDB_REPO/statements"
echo "Writable: yes"
echo "Default Graph URI: http://$INSTANCE_DOMAIN/#"
echo "Ontology Paths: (empty)"
echo "SameAs property: http://www.w3.org/2002/07/owl#sameAs"
}
printdetails
# Enable the WissKi modules.
# Enable the WissKI modules.
log_info " => Enable Wisski modules"
drush pm-enable --yes wisski_core wisski_linkblock wisski_pathbuilder wisski_adapter_sparql11_pb wisski_salz
drupal_sites_permission_workaround
# TODO: Setup WissKi-Salz.
# Because of a regresssion in EasyRDF and Tomcat, we need to manually patch EasyRDF
EASYRDF_RESPONSE="$COMPOSER_DIR/vendor/easyrdf/easyrdf/lib/EasyRdf/Http/Response.php"
log_info " => Patching '$EASYRDF_RESPONSE'"
load_template "easyrdf.patch" | patch "$EASYRDF_RESPONSE"
# Finally enable the apache2 config.
# And then reload to start serving it.
@ -184,17 +168,6 @@ log_info " => Enabling and reloading apache configuration"
a2ensite "${INSTANCE_DOMAIN}"
systemctl reload apache2
# TODO: Certbot support
# and done!
log_info " => Finished, your Drupal details are: "
echo "URL: http://$INSTANCE_DOMAIN"
echo "Username: $DRUPAL_USER"
echo "Password: $DRUPAL_PASS"
log_info " => Your GraphDB details (for WissKi Salz) are: "
echo "Read URL: http://127.0.0.1:7200/repositories/$GRAPHDB_REPO"
echo "Write URL: http://127.0.0.1:7200/repositories/$GRAPHDB_REPO/statements"
echo "Writable: yes"
echo "Default Graph URI: http://$INSTANCE_DOMAIN/owlim#"
echo "Ontology Paths: (empty)"
echo "SameAs property: http://www.w3.org/2002/07/owl#sameAs"
printdetails

5
factory/remove.sh → factory/purge.sh
View file

@ -9,7 +9,7 @@ cd "$DIR"
source "$DIR/lib/lib.sh"
require_slug_argument
# Delete the apache configurationf files first.
# Delete the apache configuration files first.
# This prevents drupal from being served.
log_info " => Removing apache configuration files"
rm "$APACHE_CONFIG_SITE_ENABLED" || true
@ -23,6 +23,7 @@ systemctl reload apache2
log_info " => Deleting MySQL database '$MYSQL_DATABASE' and user '$MYSQL_USER'. "
mysql -e "DROP DATABASE IF EXISTS \`${MYSQL_DATABASE}\`;" || true
mysql -e "DROP USER IF EXISTS \`${MYSQL_USER}\`@localhost;" || true
mysql -e "FLUSH PRIVILEGES;"
# Clear the GraphDB repository.
log_info " => Deleting GraphDB repository '$GRAPHDB_REPO'"
@ -36,4 +37,4 @@ delgroup "$SYSTEM_USER" || true
log_info " => Removing directory '$BASE_DIR'"
rm -rf "$BASE_DIR"
log_info " => Finished, '$INSTANCE_DOMAIN' has been removed. "
log_info " => Finished, '$INSTANCE_DOMAIN' has been purged. "

4
factory/resources/templates/easyrdf.patch Normal file
View file

@ -0,0 +1,4 @@
281c281
< if (preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]+)|", $status, $m)) {
---
> if(preg_match("|^HTTP/([\d\.x]+) (\d+) ([^\r\n]*)|", $status, $m)) {

57
factory/resources/templates/graphdb-repo.ttl Normal file
View file

@ -0,0 +1,57 @@
# This file is used to initialize a new GraphDB repository.
# In this file the variables ${GRAPHDB_REPO} and ${INSTANCE_DOMAIN} will be replaced.
# All other variables will be left untouched.
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
@prefix rep: <http://www.openrdf.org/config/repository#>.
@prefix sr: <http://www.openrdf.org/config/repository/sail#>.
@prefix sail: <http://www.openrdf.org/config/sail#>.
@prefix owlim: <http://www.ontotext.com/trree/owlim#>.
[] a rep:Repository ;
rep:repositoryID "${GRAPHDB_REPO}" ;
rdfs:label "${INSTANCE_DOMAIN}" ;
rep:repositoryImpl [
rep:repositoryType "graphdb:FreeSailRepository" ;
sr:sailImpl [
sail:sailType "graphdb:FreeSail" ;
owlim:owlim-license "" ;
owlim:base-URL "http://${INSTANCE_DOMAIN}/#" ;
owlim:defaultNS "" ;
owlim:entity-index-size "10000000" ;
owlim:entity-id-size "32" ;
owlim:imports "" ;
owlim:repository-type "file-repository" ;
owlim:ruleset "empty" ;
owlim:storage-folder "storage" ;
owlim:enable-context-index "false" ;
owlim:cache-memory "80m" ;
owlim:tuple-index-memory "80m" ;
owlim:enablePredicateList "false" ;
owlim:predicate-memory "0%" ;
owlim:fts-memory "0%" ;
owlim:ftsIndexPolicy "never" ;
owlim:ftsLiteralsOnly "true" ;
owlim:in-memory-literal-properties "false" ;
owlim:enable-literal-index "true" ;
owlim:index-compression-ratio "-1" ;
owlim:check-for-inconsistencies "false" ;
owlim:disable-sameAs "false" ;
owlim:enable-optimization "true" ;
owlim:transaction-mode "safe" ;
owlim:transaction-isolation "true" ;
owlim:query-timeout "0" ;
owlim:query-limit-results "0" ;
owlim:throw-QueryEvaluationException-on-timeout "false" ;
owlim:useShutdownHooks "true" ;
owlim:read-only "false" ;
owlim:nonInterpretablePredicates "http://www.w3.org/2000/01/rdf-schema#label;http://www.w3.org/1999/02/22-rdf-syntax-ns#type;http://www.ontotext.com/owlim/ces#gazetteerConfig;http://www.ontotext.com/owlim/ces#metadataConfig" ;
]
].

17
factory/resources/templates/graphdb.service Normal file
View file

@ -0,0 +1,17 @@
# This file contains a systemd service for GraphDB.
# It was tailed to a WissKI service and may or may not work in general.
[Unit]
Description=GraphDB
[Service]
Type=simple
# use the special graphdb user and group, so that other users can not just access this system.
User=graphdb
Group=graphdb
# The options here make it listen only on 127.0.0.1; this prevents external users from attempting access.
ExecStart=/opt/graphdb/bin/graphdb –Xmx6g -Dgraphdb.connector.address=127.0.0.1
[Install]
WantedBy=multi-user.target

16
factory/resources/templates/wisski-site.conf Normal file
View file

@ -0,0 +1,16 @@
# This file was generated automatically.
# It contains configuration for the WissKI site:
# ${INSTANCE_DOMAIN}
# Do not edit unless you know exactly what you are doing.
<VirtualHost *:${PUBLIC_PORT}>
DocumentRoot ${WEB_DIR}
ServerName ${INSTANCE_DOMAIN}
AssignUserId ${SYSTEM_USER} ${SYSTEM_USER}
<Directory ${WEB_DIR}>
include "${WISSKI_COMMON_PATH}/wisski-common-dir.conf"
</Directory>
include "${WISSKI_COMMON_PATH}/wisski-common.conf"
</VirtualHost>

14
factory/resources/wisski-apache-common/wisski-common-dir.conf Normal file
View file

@ -0,0 +1,14 @@
# This file is included inside the 'directory' configuration of all WissKI sites.
# add types for .owl and .rdf
AddType application/rdf+xml .owl
AddType application/rdf+xml .rdf
# Rewrites the ontology directory
ReWriteRule ^(ontology/[^/]+/).+ $1 [R=303,L]
ReWriteRule ^(ontology/[^/]+)/$ sites/default/files/$1.owl [L]
# Allow overrides of symlinks
Options Indexes FollowSymLinks
AllowOverride All
Require all granted

4
factory/resources/wisski-apache-common/wisski-common.conf Normal file
View file

@ -0,0 +1,4 @@
# This file is included inside the 'virtualhost' of a WissKI site.
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined

25
factory/system_install.sh
View file

@ -35,7 +35,7 @@ apt-get install -y curl php-cli php-mbstring git unzip
curl -sS https://getcomposer.org/installer -o "$tmpdir/composer-setup.php"
php $tmpdir/composer-setup.php --install-dir=/usr/local/bin --filename=composer
# Install required php extensions for Drupal and WissKi.
# Install required php extensions for Drupal and WissKI.
log_info " => Installing required php extensions"
apt-get install -y php-xml php-gd php-mysql php-common php-xmlrpc php-soap php-gd php-intl php-mysql php-zip php-curl php-ssh2
@ -47,6 +47,15 @@ apt-get -y install mariadb-server
log_info " => Installing apache2, php and auth modules"
apt-get install -y apache2 libapache2-mod-php libapache2-mpm-itk
# Install apache and required php extensions.
log_info " => Enabling apache 'rewrite' module"
a2enmod rewrite
# Copy over the wisski templates
log_info " => Copying over Apache Templates"
mkdir -p "$WISSKI_COMMON_PATH"
cp -v $SCRIPT_DIR/resources/wisski-apache-common/* "$WISSKI_COMMON_PATH"
# Make the directory for all drupal instances to live in.
log_info " => Making root directory for Drupal Installations"
mkdir -p "$DRUPAL_ROOT"
@ -85,19 +94,7 @@ chown -R graphdb:graphdb /opt/graphdb
# This file uses the users created above, and also hard-codes listening address and maximum memory.
# This avoids having to write the config file using bash hacks.
log_info " => Making 'graphdb.service'"
cat << "EOF" > /etc/systemd/system/graphdb.service
[Unit]
Description=GraphDB
[Service]
Type=simple
User=graphdb
Group=graphdb
ExecStart=/opt/graphdb/bin/graphdb –Xmx6g -Dgraphdb.connector.address=127.0.0.1
[Install]
WantedBy=multi-user.target
EOF
load_template 'graphdb.service' > /etc/systemd/system/graphdb.service
# We just created a service, so now start it and put it into autostart mode.
log_info " => Starting and enabling graphdb.service"