rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move code into new component package

Browse source 
This commit cleans up the resources in the 'embed' package, and instead
moves them into subpackages of a new 'compose' package. This makes sure
that '.env' templates and docker compose contexts are located in the
same location.
This commit is contained in:
Tom Wiesing 2022-09-11 15:41:11 +02:00
parent 2ee90bf462
commit 7b2f79bea1
No known key found for this signature in database
44 changed files with 579 additions and 559 deletions
Download patch file Download diff file Expand all files Collapse all files

118
embed/install.go
View file

@ -1,118 +0,0 @@
// Package embed contains embedded resources
package embed
import (
"embed"
"io"
"io/fs"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// ResourceEmbed contains all the resources required by the WissKI-Distillery package.
//go:embed all:resources
var ResourceEmbed embed.FS
// InstallResource install a resource src into dest.
// When it encounters a directory, recursively installs the directory is called.
// For each installation item, onInstallFile is called, unless onInstallFile is nil.
//
// If src points to a file, dst must either be an existing file, or not exist.
// If src points to a directory, dst must either be an existing directory, or not exist.
func InstallResource(dst, src string, onInstallFile func(dst, src string)) error {
return installFile(dst, ResourceEmbed, src, onInstallFile)
}
var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory")
var errExpectedDirectoryButGotFile = errors.New("Expected a directory, but got a file")
func installFile(dst string, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// call the on-install file path
if onInstallFile != nil {
onInstallFile(dst, src)
}
// open the source file!
srcFile, err := fsys.Open(src)
if err != nil {
return errors.Wrapf(err, "Error opening source file %s", src)
}
defer srcFile.Close()
// stat the source file to install
srcStat, srcErr := srcFile.Stat()
if srcErr != nil {
return errors.Wrapf(srcErr, "Error calling stat on source %s", src)
}
// if it is a directory, we should recurse!
if srcStat.IsDir() {
return installDir(dst, srcStat, srcFile, fsys, src, onInstallFile)
}
// determine if we need to create the destination file, or if it already exists
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case dstStat.IsDir():
return errors.Wrapf(errExpectedFileButGotDirectory, "Error processing destination %s", dst)
}
// Open the file
dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcStat.Mode())
if err != nil {
return errors.Wrapf(err, "Error opening destination %s", dst)
}
defer dstFile.Close()
// copy over the content
_, err = io.Copy(dstFile, srcFile)
return errors.Wrapf(err, "Error writing to destination %s", dst)
}
func installDir(dst string, srcStat fs.FileInfo, srcFile fs.File, fsys embed.FS, src string, onInstallFile func(dst, src string)) error {
// make sure it is a directory!
dir, ok := srcFile.(fs.ReadDirFile)
if !ok {
return errExpectedDirectoryButGotFile
}
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcStat.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case !dstStat.IsDir():
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
case dstErr == nil:
}
// read the directory
entries, err := dir.ReadDir(-1)
if err != nil {
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
}
// iterate over all the children
for _, entry := range entries {
if err := func(dst, src string) error {
return installFile(dst, fsys, src, onInstallFile)
}(
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
); err != nil {
return err
}
}
return nil
}

10
embed/legacy.go Normal file
View file

@ -0,0 +1,10 @@
// Package embed contains embedded resources
package embed
import (
"embed"
)
// ResourceEmbed contains all the resources required by the WissKI-Distillery package.
//go:embed all:resources
var ResourceEmbed embed.FS

1
embed/paths.go
View file

@ -1 +0,0 @@
package embed

5
embed/resources/compose/dis/Dockerfile
View file

@ -1,5 +0,0 @@
FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","dis_server","--bind","0.0.0.0:8888"]

28
embed/resources/compose/dis/docker-compose.yml
View file

@ -1,28 +0,0 @@
version: "3.7"
services:
wdresolve:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /dis/
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
networks:
default:
name: distillery
external: true

5
embed/resources/compose/resolver/Dockerfile
View file

@ -1,5 +0,0 @@
FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]

29
embed/resources/compose/resolver/docker-compose.yml
View file

@ -1,29 +0,0 @@
version: "3.7"
services:
wdresolve:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /go/
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
- "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
networks:
default:
name: distillery
external: true

28
embed/resources/compose/self/docker-compose.yml
View file

@ -1,28 +0,0 @@
version: "3.7"
services:
tr:
image: ghcr.io/tkw1536/tr:latest
restart: always
volumes:
- "${OVERRIDES_FILE}:/overrides.json:ro"
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8080
VIRTUAL_PATH: /
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
# the overrides file
OVERRIDES: /overrides.json
# where to redirect to
TARGET: ${TARGET}
networks:
default:
name: distillery
external: true

35
embed/resources/compose/sql/docker-compose.yml
View file

@ -1,35 +0,0 @@
version: "3.7"
services:
sql:
image: mariadb
volumes:
- "./data/:/var/lib/mysql"
ports:
- 127.0.0.1:3306:3306
environment:
# This combination of environment variables will configure a passwordless root user
# that can only connect to the container from 'localhost'.
# This means we can only connect using 'docker-compose exec sql mysql -C '...' '.
- "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
- "MYSQL_ROOT_HOST=localhost"
restart: always
phpmyadmin:
image: phpmyadmin/phpmyadmin
environment:
- "PMA_HOST=sql"
- "HIDE_PHP_VERSION=true"
- "UPLOAD_LIMIT=100M"
# phpmyadmin running on localhost:8080 so that we can easily access the system graphically.
# By default no admin account is created, so initial shell access to make one is needed.
ports:
- 127.0.0.1:8080:80
depends_on:
- sql
restart: always
networks:
default:
name: distillery
external: true

17
embed/resources/compose/ssh/docker-compose.yml
View file

@ -1,17 +0,0 @@
version: "3.7"
services:
ssh:
image: ghcr.io/tkw1536/dockersshd:latest
command: -hostkey /keys/hostkey -shell /user_shell.sh -keylabel eu.wiss-ki.barrel.authfile -userlabel eu.wiss-ki.barrel.slug -L triplestore:7200 -L phpmyadmin:80 -L sql:3306
ports:
- "2222:2222"
volumes:
- './data/keys:/keys'
- '/var/run/docker.sock:/var/run/docker.sock:ro'
restart: always
networks:
default:
name: distillery
external: true

3
embed/resources/compose/triplestore/.dockerignore
View file

@ -1,3 +0,0 @@
*
!*.zip
!entrypoint.sh

64
embed/resources/compose/triplestore/Dockerfile
View file

@ -1,64 +0,0 @@
# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container.
# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
# but has been modified for performance and security.
# This image is intended to be built like:
# docker build --build-arg graphdb_src=graphdb.zip .
# We first make a base image to base further builds on.
# We don't use alpine here, as that uses significantly slower musl instead of glibc.
FROM adoptopenjdk/openjdk11:debian-slim as base
# Create a user called graphdb
RUN useradd -ms /bin/bash graphdb
# make a base images, to add the sources to.
FROM base as sources
# install unzip
RUN apt-get update && apt-get install -y unzip
# add the source file (by default graphdb.zip) to the image
ARG src=graphdb.zip
ADD ${src} /graphdb.zip
# unpack it into a temporary directory
RUN unzip "$src" -d "/unpack/"
# Move it into /opt/graphdb, and chown it to graphdb
RUN mv "/unpack"/* /opt/graphdb
RUN chown -R graphdb:graphdb /opt/graphdb
# finally make an image that will run
FROM base as final
# add the entrypoint script
ADD entrypoint.sh /entrypoint.sh
# copy over the sources
COPY --from=sources /opt/graphdb /opt/graphdb
# set environment variables for graphdb_home and path
ENV GRAPHDB_HOME=/opt/graphdb
ENV PATH=$GRAPHDB_HOME/bin:$PATH
# Workaround for CVE-2021-44228
# (not sure if we are vulnerable, but just because)
ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
# expose a port
EXPOSE 7200
# setup a healthcheck, that checks if the server is up.
RUN apt-get update && apt-get install -y curl
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
# Add volumes for data, work and logs as these might be accessible from the outside.
# To add your own configuration, manually mount a config file into /opt/graphdb/work
VOLUME /opt/graphdb/data
VOLUME /opt/graphdb/work
VOLUME /opt/graphdb/logs
# setup command and entrypoint
CMD ["-Dgraphdb.home=/opt/graphdb"]
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

22
embed/resources/compose/triplestore/docker-compose.yml
View file

@ -1,22 +0,0 @@
version: "3.7"
services:
triplestore:
build: .
ports:
- "127.0.0.1:7200:7200"
volumes:
- './data/data:/opt/graphdb/data'
- './data/work:/opt/graphdb/work'
- './data/logs:/opt/graphdb/logs'
command: "\"-Dgraphdb.home=/opt/graphdb -Ddefault.min.distinct.threshold=2G\""
# Use 1GB of heap space
environment:
GDB_HEAP_SIZE: 16G
restart: always
networks:
default:
name: distillery
external: true

13
embed/resources/compose/triplestore/entrypoint.sh
View file

@ -1,13 +0,0 @@
#!/bin/bash
set -e
# Because we want to run graphdb as a limited user
# we need to make sure that the volumes are writable.
# Because of that, we 'chown'
chown graphdb:graphdb /opt/graphdb/data
chown graphdb:graphdb /opt/graphdb/work
chown graphdb:graphdb /opt/graphdb/logs
# switch to the graphdb user, and run graphdb
su graphdb -c "/opt/graphdb/bin/graphdb $@"

52
embed/resources/compose/web/docker-compose.yml
View file

@ -1,52 +0,0 @@
version: "3.7"
services:
nginx-proxy:
image: ghcr.io/nginx-proxy/nginx-proxy:alpine
environment:
- DEFAULT_HOST=${DEFAULT_HOST}
- HTTPS_METHOD=${HTTPS_METHOD}
ports:
- "80:80"
- "443:443"
volumes:
- "vhost:/etc/nginx/vhost.d"
- "./global.conf:/etc/nginx/conf.d/global.conf:ro"
- "./proxy.conf:/etc/nginx/proxy.conf:ro"
- "htpasswd:/etc/nginx/htpasswd"
- "html:/usr/share/nginx/html"
- "/var/run/docker.sock:/tmp/docker.sock:ro"
- "certs:/etc/nginx/certs"
labels:
com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy: true
restart: always
networks:
- default
letsencrypt-nginx-proxy-companion:
image: docker.io/nginxproxy/acme-companion:latest
volumes:
- "/var/run/docker.sock:/var/run/docker.sock:ro"
- "htpasswd:/etc/nginx/htpasswd"
- "vhost:/etc/nginx/vhost.d"
- "html:/usr/share/nginx/html"
- "/var/run/docker.sock:/tmp/docker.sock:ro"
- "certs:/etc/nginx/certs"
- "acme:/etc/acme.sh"
restart: always
networks:
- default
depends_on:
- nginx-proxy
volumes:
acme:
vhost:
html:
certs:
htpasswd:
networks:
default:
name: distillery
external: true

4
embed/resources/compose/web/global.conf
View file

@ -1,4 +0,0 @@
# Nginx Configuration File
# These should match with distillery/resources/compose/barrel/conf/wisski.ini.
client_max_body_size 1000m;

19
embed/resources/compose/web/proxy.conf
View file

@ -1,19 +0,0 @@
# HTTP 1.1 support
proxy_http_version 1.1;
proxy_buffering off;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $proxy_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl;
proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port;
# Mitigate httpoxy attack (see README for details)
proxy_set_header Proxy "";
# Timeouts for the proxy connection - in sync with the appropriate max_execution time.
proxy_connect_timeout 3000s;
proxy_read_timeout 3000s;
proxy_send_timeout 3000s;

9
embed/resources/templates/docker-env/dis
View file

@ -1,9 +0,0 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}

10
embed/resources/templates/docker-env/resolver
View file

@ -1,10 +0,0 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
RESOLVER_CONFIG=${RESOLVER_CONFIG}

7
embed/resources/templates/docker-env/self
View file

@ -1,7 +0,0 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
TARGET=${TARGET}
OVERRIDES_FILE=${OVERRIDES_FILE}

2
embed/resources/templates/docker-env/web
View file

@ -1,2 +0,0 @@
DEFAULT_HOST=${DEFAULT_HOST}
HTTPS_METHOD=${HTTPS_METHOD}