Move code into new component package
This commit cleans up the resources in the 'embed' package, and instead moves them into subpackages of a new 'compose' package. This makes sure that '.env' templates and docker compose contexts are located in the same location.
This commit is contained in:
Tom Wiesing
parent
2ee90bf462
commit
7b2f79bea1
44 changed files with 579 additions and 559 deletions
315
component/triplestore/database.go
Normal file
315
component/triplestore/database.go
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
package triplestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/embed"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/logging"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/unpack"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/wait"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tkw1536/goprogram/exit"
|
||||
"github.com/tkw1536/goprogram/stream"
|
||||
)
|
||||
|
||||
type TriplestoreUserPayload struct {
|
||||
Password string `json:"password"`
|
||||
AppSettings TriplestoreUserAppSettings `json:"appSettings"`
|
||||
GrantedAuthorities []string `json:"grantedAuthorities"`
|
||||
}
|
||||
type TriplestoreUserAppSettings struct {
|
||||
DefaultInference bool `json:"DEFAULT_INFERENCE"`
|
||||
DefaultVisGraphSchema bool `json:"DEFAULT_VIS_GRAPH_SCHEMA"`
|
||||
DefaultSameas bool `json:"DEFAULT_SAMEAS"`
|
||||
IgnoreSharedQueries bool `json:"IGNORE_SHARED_QUERIES"`
|
||||
ExecuteCount bool `json:"EXECUTE_COUNT"`
|
||||
}
|
||||
|
||||
// OpenRaw makes an http request to the triplestore api.
|
||||
//
|
||||
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
|
||||
// When bodyName is empty, simply marshal body as application/json
|
||||
func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName string, accept string) (*http.Response, error) {
|
||||
var reader io.Reader
|
||||
|
||||
var contentType string
|
||||
|
||||
// for "PUT" and "POST" we setup a body
|
||||
if method == "PUT" || method == "POST" {
|
||||
if bodyName != "" {
|
||||
buffer := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(buffer)
|
||||
contentType = writer.FormDataContentType()
|
||||
|
||||
part, err := writer.CreateFormFile(bodyName, "filename.txt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
io.Copy(part, bytes.NewReader(body.([]byte)))
|
||||
writer.Close()
|
||||
reader = buffer
|
||||
} else {
|
||||
contentType = "application/json"
|
||||
mbytes, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader = bytes.NewReader(mbytes)
|
||||
}
|
||||
}
|
||||
|
||||
// create the request object
|
||||
req, err := http.NewRequest(method, ts.BaseURL+url, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Setup configuration!
|
||||
if accept != "" {
|
||||
req.Header.Set("Accept", accept)
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
req.SetBasicAuth(ts.Config.TriplestoreAdminUser, ts.Config.TriplestoreAdminPassword)
|
||||
|
||||
// and send it
|
||||
return http.DefaultClient.Do(req)
|
||||
}
|
||||
|
||||
// Wait waits for the connection to the Triplestore to succeed.
|
||||
// This is achieved using a polling strategy.
|
||||
func (ts Triplestore) Wait() error {
|
||||
return wait.Wait(func() bool {
|
||||
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer res.Body.Close()
|
||||
return true
|
||||
}, ts.PollInterval, ts.PollContext)
|
||||
}
|
||||
|
||||
var errTripleStoreFailedRepository = exit.Error{
|
||||
Message: "Failed to create repository: %s",
|
||||
ExitCode: exit.ExitGeneric,
|
||||
}
|
||||
|
||||
func (ts Triplestore) Provision(name, domain, user, password string) error {
|
||||
if err := ts.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// prepare the create repo request
|
||||
// TODO: Move this into a seperate file
|
||||
createRepo, _, err := unpack.UnpackTemplate(
|
||||
map[string]string{
|
||||
"GRAPHDB_REPO": name,
|
||||
"INSTANCE_DOMAIN": domain,
|
||||
},
|
||||
fsx.OpenFS(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), embed.ResourceEmbed),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do the create!
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/repositories", createRepo, "config", "")
|
||||
if err != nil {
|
||||
return errTripleStoreFailedRepository.WithMessageF(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusCreated {
|
||||
return errTripleStoreFailedRepository.WithMessageF("Repo create did not return status code 201")
|
||||
}
|
||||
}
|
||||
|
||||
// create the user and grant them access
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/security/users/"+user, TriplestoreUserPayload{
|
||||
Password: password,
|
||||
AppSettings: TriplestoreUserAppSettings{
|
||||
DefaultInference: true,
|
||||
DefaultVisGraphSchema: true,
|
||||
DefaultSameas: true,
|
||||
IgnoreSharedQueries: false,
|
||||
ExecuteCount: true,
|
||||
},
|
||||
GrantedAuthorities: []string{
|
||||
"ROLE_USER",
|
||||
"READ_REPO_" + name,
|
||||
"WRITE_REPO_" + name,
|
||||
},
|
||||
}, "", "")
|
||||
if err != nil {
|
||||
return errTripleStoreFailedRepository.WithMessageF(err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusCreated {
|
||||
return errTripleStoreFailedRepository.WithMessageF("User create did not return status code 201")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// TriplestorePurgeUser deletes the specified user from the triplestore
|
||||
func (ts Triplestore) PurgeUser(user string) error {
|
||||
res, err := ts.OpenRaw("DELETE", "/rest/security/users/"+user, nil, "", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != http.StatusNoContent {
|
||||
return errors.Errorf("Delete returned code %d", res.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TriplestorePurgeRepo deletes the specified repo from the triplestore
|
||||
func (ts Triplestore) PurgeRepo(repo string) error {
|
||||
res, err := ts.OpenRaw("DELETE", "/rest/repositories/"+repo, nil, "", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return errors.Errorf("Delete returned code %d", res.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errTSBackupWrongStatusCode = errors.New("Distillery.Backup: Wrong status code")
|
||||
|
||||
// TriplestoreBackup backs up the repository named repo into the writer dst.
|
||||
func (ts Triplestore) Backup(dst io.Writer, repo string) (int64, error) {
|
||||
res, err := ts.OpenRaw("GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return 0, errTSBackupWrongStatusCode
|
||||
}
|
||||
defer res.Body.Close()
|
||||
return io.Copy(dst, res.Body)
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
URI string `json:"uri"`
|
||||
Type string `json:"type"`
|
||||
SesameType string `json:"sesameType"`
|
||||
Location string `json:"location"`
|
||||
Readable bool `json:"readable"`
|
||||
Writable bool `json:"writable"`
|
||||
Local bool `json:"local"`
|
||||
}
|
||||
|
||||
func (ts Triplestore) listRepositories() (repos []Repository, err error) {
|
||||
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "application/json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
err = json.NewDecoder(res.Body).Decode(&repos)
|
||||
return
|
||||
}
|
||||
|
||||
// TriplestoreBackup backs up every graphdb instance into dst
|
||||
func (ts Triplestore) BackupAll(dst string) error {
|
||||
// list all the repositories
|
||||
repos, err := ts.listRepositories()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the base directory
|
||||
if err := os.Mkdir(dst, fs.ModeDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// iterate over all the repositories
|
||||
for _, repo := range repos {
|
||||
if rErr := (func(repo Repository) error {
|
||||
name := filepath.Join(dst, repo.ID+".nq")
|
||||
|
||||
dest, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dest.Close()
|
||||
|
||||
_, err = ts.Backup(dest, repo.ID)
|
||||
return err
|
||||
}(repo)); err == nil && rErr != nil {
|
||||
err = rErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
|
||||
|
||||
func (ts Triplestore) Bootstrap(io stream.IOStream) error {
|
||||
logging.LogMessage(io, "Waiting for Triplestore")
|
||||
if err := ts.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logging.LogMessage(io, "Resetting admin user password")
|
||||
{
|
||||
res, err := ts.OpenRaw("PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
|
||||
Password: ts.Config.TriplestoreAdminPassword,
|
||||
AppSettings: TriplestoreUserAppSettings{
|
||||
DefaultInference: true,
|
||||
DefaultVisGraphSchema: true,
|
||||
DefaultSameas: true,
|
||||
IgnoreSharedQueries: false,
|
||||
ExecuteCount: true,
|
||||
},
|
||||
GrantedAuthorities: []string{"ROLE_ADMIN"},
|
||||
}, "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create triplestore user: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK:
|
||||
// we set the password => requests are unauthorized
|
||||
// so we still need to enable security (see below!)
|
||||
case http.StatusUnauthorized:
|
||||
// a password is needed => security is already enabled.
|
||||
// the password may or may not work, but that's a problem for later
|
||||
logging.LogMessage(io, "Security is already enabled")
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("failed to create triplestore user: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
logging.LogMessage(io, "Enabling Triplestore security")
|
||||
{
|
||||
res, err := ts.OpenRaw("POST", "/rest/security", true, "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to enable triplestore security: %s", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return errTriplestoreFailedSecurity
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
3
component/triplestore/stack/.dockerignore
Normal file
3
component/triplestore/stack/.dockerignore
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
*
|
||||
!*.zip
|
||||
!entrypoint.sh
|
||||
64
component/triplestore/stack/Dockerfile
Normal file
64
component/triplestore/stack/Dockerfile
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
# This Dockerfile contains instructions to compile and run GraphDB inside a Docker container.
|
||||
# It is roughly based on https://github.com/Ontotext-AD/graphdb-docker/blob/master/free-edition/Dockerfile
|
||||
# but has been modified for performance and security.
|
||||
|
||||
# This image is intended to be built like:
|
||||
# docker build --build-arg graphdb_src=graphdb.zip .
|
||||
|
||||
# We first make a base image to base further builds on.
|
||||
# We don't use alpine here, as that uses significantly slower musl instead of glibc.
|
||||
FROM adoptopenjdk/openjdk11:debian-slim as base
|
||||
|
||||
# Create a user called graphdb
|
||||
RUN useradd -ms /bin/bash graphdb
|
||||
|
||||
# make a base images, to add the sources to.
|
||||
FROM base as sources
|
||||
|
||||
# install unzip
|
||||
RUN apt-get update && apt-get install -y unzip
|
||||
|
||||
# add the source file (by default graphdb.zip) to the image
|
||||
ARG src=graphdb.zip
|
||||
ADD ${src} /graphdb.zip
|
||||
|
||||
# unpack it into a temporary directory
|
||||
RUN unzip "$src" -d "/unpack/"
|
||||
|
||||
# Move it into /opt/graphdb, and chown it to graphdb
|
||||
RUN mv "/unpack"/* /opt/graphdb
|
||||
RUN chown -R graphdb:graphdb /opt/graphdb
|
||||
|
||||
# finally make an image that will run
|
||||
FROM base as final
|
||||
|
||||
# add the entrypoint script
|
||||
ADD entrypoint.sh /entrypoint.sh
|
||||
|
||||
# copy over the sources
|
||||
COPY --from=sources /opt/graphdb /opt/graphdb
|
||||
|
||||
# set environment variables for graphdb_home and path
|
||||
ENV GRAPHDB_HOME=/opt/graphdb
|
||||
ENV PATH=$GRAPHDB_HOME/bin:$PATH
|
||||
|
||||
# Workaround for CVE-2021-44228
|
||||
# (not sure if we are vulnerable, but just because)
|
||||
ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
|
||||
|
||||
# expose a port
|
||||
EXPOSE 7200
|
||||
|
||||
# setup a healthcheck, that checks if the server is up.
|
||||
RUN apt-get update && apt-get install -y curl
|
||||
HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD curl --fail 127.0.0.1:7200/rest/repositories || exit 1
|
||||
|
||||
# Add volumes for data, work and logs as these might be accessible from the outside.
|
||||
# To add your own configuration, manually mount a config file into /opt/graphdb/work
|
||||
VOLUME /opt/graphdb/data
|
||||
VOLUME /opt/graphdb/work
|
||||
VOLUME /opt/graphdb/logs
|
||||
|
||||
# setup command and entrypoint
|
||||
CMD ["-Dgraphdb.home=/opt/graphdb"]
|
||||
ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
|
||||
22
component/triplestore/stack/docker-compose.yml
Normal file
22
component/triplestore/stack/docker-compose.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
version: "3.7"
|
||||
|
||||
services:
|
||||
triplestore:
|
||||
build: .
|
||||
ports:
|
||||
- "127.0.0.1:7200:7200"
|
||||
volumes:
|
||||
- './data/data:/opt/graphdb/data'
|
||||
- './data/work:/opt/graphdb/work'
|
||||
- './data/logs:/opt/graphdb/logs'
|
||||
command: "\"-Dgraphdb.home=/opt/graphdb -Ddefault.min.distinct.threshold=2G\""
|
||||
# Use 1GB of heap space
|
||||
environment:
|
||||
GDB_HEAP_SIZE: 16G
|
||||
|
||||
restart: always
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: distillery
|
||||
external: true
|
||||
13
component/triplestore/stack/entrypoint.sh
Normal file
13
component/triplestore/stack/entrypoint.sh
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Because we want to run graphdb as a limited user
|
||||
# we need to make sure that the volumes are writable.
|
||||
# Because of that, we 'chown'
|
||||
|
||||
chown graphdb:graphdb /opt/graphdb/data
|
||||
chown graphdb:graphdb /opt/graphdb/work
|
||||
chown graphdb:graphdb /opt/graphdb/logs
|
||||
|
||||
# switch to the graphdb user, and run graphdb
|
||||
su graphdb -c "/opt/graphdb/bin/graphdb $@"
|
||||
44
component/triplestore/triplestore.go
Normal file
44
component/triplestore/triplestore.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package triplestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/FAU-CDI/wisski-distillery/component"
|
||||
"github.com/FAU-CDI/wisski-distillery/internal/stack"
|
||||
)
|
||||
|
||||
type Triplestore struct {
|
||||
component.ComponentBase
|
||||
|
||||
BaseURL string // upstream server url
|
||||
|
||||
PollContext context.Context // context to abort polling with
|
||||
PollInterval time.Duration // duration to wait for during wait
|
||||
}
|
||||
|
||||
func (Triplestore) Name() string {
|
||||
return "triplestore"
|
||||
}
|
||||
|
||||
//go:embed all:stack
|
||||
var resources embed.FS
|
||||
|
||||
func (ts Triplestore) Stack() stack.Installable {
|
||||
return ts.ComponentBase.MakeStack(stack.Installable{
|
||||
Resources: resources,
|
||||
ContextPath: "stack",
|
||||
|
||||
CopyContextFiles: []string{"graphdb.zip"}, // TODO: Move into constant?
|
||||
|
||||
MakeDirsPerm: fs.ModeDir | fs.ModePerm,
|
||||
MakeDirs: []string{
|
||||
filepath.Join("data", "data"),
|
||||
filepath.Join("data", "work"),
|
||||
filepath.Join("data", "logs"),
|
||||
},
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue