resolver: Use self-built image

This commit updates the resolver component to use an image that is built
locally.

README.md

@@ -79,7 +79,7 @@ These are:
 
   - This will delegate individual hostnames to appropriate docker containers, see [this blog post](http://jasonwilder.com/blog/2014/03/25/automated-nginx-reverse-proxy-for-docker/) for an overview.
   - Optionally makes use of [docker-letsencrypt-nginx-proxy-companion](https://github.com/nginx-proxy/docker-letsencrypt-nginx-proxy-companion) to automatically provision and renew HTTPS certificates.
-  - See [distillery/resources/compose/web](distillery/resources/compose/web) for implementation details.
+  - See [distillery/resources/compose/web](embed/resources/compose/web) for implementation details.
 
 - [MariaDB](https://mariadb.org/) - an SQL server
 
@@ -90,25 +90,30 @@ These are:
   - It is accsssible using `127.0.0.1:3306`
   - A database shell can be opened using `sudo /var/www/deploy/wdcli mysql`.
   - A [phpmyadmin](https://www.phpmyadmin.net/) is started on `127.0.0.1:8080`.
-  - See [distillery/resources/compose/sql](distillery/resources/compose/sql) for implementation details.
+  - See [distillery/resources/compose/sql](embed/resources/compose/sql) for implementation details.
 
 - [GraphDB](http://graphdb.ontotext.com/) - a SPARQL backend for WissKI (Version 10.0 or later)
   - It is configured to run inside a docker container.
   - The Workbench API is started on `127.0.0.1:7200`.
   - Security is not enabled at the moment.
-  - See [distillery/resources/compose/triplestore](distillery/resources/compose/triplestore) for implementation details.
+  - See [distillery/resources/compose/triplestore](embed/resources/compose/triplestore) for implementation details.
 
 - [proxyssh](https://github.com/tkw1536/proxyssh) - an ssh server that delegates client connections to different WissKIs
   - It is configured to run inside a docker container.
   - Uses a global configurable authorized_keys file.
   - Also allows users to write their own authorized_keys files.
-  - See [distillery/resources/compose/ssh](distillery/resources/compose/ssh) for implementation details.
+  - See [distillery/resources/compose/ssh](embed/resources/compose/ssh) for implementation details.
 
 - [wdresolve](https://github.com/FAU-CDI/wdresolve) - a global WissKI Distillery Resolver
   - It is configured to run inside a docker container
   - Uses configuration which is updated with `sudo /var/www/deploy/wdcli update_prefix_config` 
   - Running in the browser under the `/go/` path of the main domain.
-  - See [distillery/resources/compose/resolver](distillery/resources/compose/resolver) for implementation details.
+  - See [distillery/resources/compose/resolver](embed/resources/compose/resolver) for implementation details.
+
+- `dis` - a WissKI Distillery Information Server
+  - It is configured to run inside a docker container
+  - Running in the browser under the `/dis/` path of the main domain.
+  - See [distillery/resources/compose/resolver](embed/resources/compose/dis) for implementation details.
 
 To manage multiple docker containers, this script makes heavy use of [docker compose](https://docs.docker.com/compose/).
 

TODO.md

@@ -10,7 +10,8 @@
 - Enhance Snapshots
     - Export the Docker Images
 - Avoid running `docker compose` executable and shift it to a library
+- Move resolver code into this
 - Cleanup code: Have consistent error handling
-- Add a metadata / statistics server
+- Add a proper metadata / statistics server
 - Restructure code files
 - Single Malt Mode: Support having a single instance only!

cmd/bootstrap.go

@@ -7,7 +7,7 @@ import (
 
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/embed"
 	cfg "github.com/FAU-CDI/wisski-distillery/internal/config"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/FAU-CDI/wisski-distillery/internal/hostname"
@@ -123,7 +123,7 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
 					return errBootstrapWriteConfig.WithMessageF(err)
 				}
 
-				if err := distillery.InstallTemplate(envPath, filepath.Join("resources", "templates", "bootstrap", "env"), map[string]string{
+				if err := embed.InstallTemplate(envPath, filepath.Join("resources", "templates", "bootstrap", "env"), map[string]string{
 					"DEPLOY_ROOT":          root,
 					"DEFAULT_DOMAIN":       domain,
 					"SELF_OVERRIDES_FILE":  overridesPath,
@@ -146,12 +146,12 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error {
 			if err := logging.LogOperation(func() error {
 
 				context.Println(overridesPath)
-				if err := distillery.InstallTemplate(overridesPath, filepath.Join("resources", "templates", "bootstrap", "overrides.json"), map[string]string{}); err != nil {
+				if err := embed.InstallTemplate(overridesPath, filepath.Join("resources", "templates", "bootstrap", "overrides.json"), map[string]string{}); err != nil {
 					return errBootstrapCreateFile.WithMessageF(err)
 				}
 
 				context.Println(authorizedKeysFile)
-				if err := distillery.InstallTemplate(authorizedKeysFile, filepath.Join("resources", "templates", "bootstrap", "global_authorized_keys"), map[string]string{}); err != nil {
+				if err := embed.InstallTemplate(authorizedKeysFile, filepath.Join("resources", "templates", "bootstrap", "global_authorized_keys"), map[string]string{}); err != nil {
 					return errBootstrapCreateFile.WithMessageF(err)
 				}
 

cmd/dis_server.go

@@ -1,43 +0,0 @@
-package cmd
-
-import (
-	"net/http"
-
-	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
-	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/tkw1536/goprogram/exit"
-)
-
-// DisServer is the 'dis_server' command
-var DisServer wisski_distillery.Command = disServer{}
-
-type disServer struct {
-	Prefix string `short:"p" long:"prefix" description:"prefix to listen under"`
-	Bind   string `short:"b" long:"bind" description:"address to listen on" default:"127.0.0.1:8888"`
-}
-
-func (disServer) Description() wisski_distillery.Description {
-	return wisski_distillery.Description{
-		Requirements: core.Requirements{
-			NeedsDistillery: true,
-		},
-		Command:     "dis_server",
-		Description: "Starts a server with information about this distillery",
-	}
-}
-
-var errServerListen = exit.Error{
-	ExitCode: exit.ExitGeneric,
-	Message:  "Unable to listen",
-}
-
-func (s disServer) Run(context wisski_distillery.Context) error {
-	server := context.Environment.Server()
-
-	context.Printf("Listening on %s\n", s.Bind)
-	err := http.ListenAndServe(s.Bind, http.StripPrefix(s.Prefix, server))
-	if err == nil {
-		return nil
-	}
-	return errServerListen.Wrap(err)
-}

cmd/servers.go

@@ -0,0 +1,68 @@
+package cmd
+
+import (
+	"net/http"
+
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/core"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// ResolverServer is the 'resolver_server' command
+var ResolverServer wisski_distillery.Command = server{
+	Desc: wisski_distillery.Description{
+		Requirements: core.Requirements{
+			NeedsDistillery: true,
+		},
+		Command:     "resolver_server",
+		Description: "Starts a global resolver server",
+	},
+	Server: func(context wisski_distillery.Context) (http.Handler, error) {
+		return context.Environment.Resolver().Server(context.IOStream)
+	},
+}
+
+// DisServer is the 'dis_server' command
+var DisServer wisski_distillery.Command = server{
+	Desc: wisski_distillery.Description{
+		Requirements: core.Requirements{
+			NeedsDistillery: true,
+		},
+		Command:     "dis_server",
+		Description: "Starts a server with information about this distillery",
+	},
+	Server: func(context wisski_distillery.Context) (http.Handler, error) {
+		return context.Environment.Server(), nil
+	},
+}
+
+type server struct {
+	Prefix string `short:"p" long:"prefix" description:"prefix to listen under"`
+	Bind   string `short:"b" long:"bind" description:"address to listen on" default:"127.0.0.1:8888"`
+
+	Desc   wisski_distillery.Description
+	Server func(context wisski_distillery.Context) (http.Handler, error)
+}
+
+func (s server) Description() wisski_distillery.Description {
+	return s.Desc
+}
+
+var errServerListen = exit.Error{
+	ExitCode: exit.ExitGeneric,
+	Message:  "Unable to listen",
+}
+
+func (s server) Run(context wisski_distillery.Context) error {
+	handler, err := s.Server(context)
+	if err != nil {
+		return err
+	}
+
+	context.Printf("Listening on %s\n", s.Bind)
+	err = http.ListenAndServe(s.Bind, http.StripPrefix(s.Prefix, handler))
+	if err == nil {
+		return nil
+	}
+	return errServerListen.Wrap(err)
+}

cmd/system_update.go

@@ -6,7 +6,7 @@ import (
 
 	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
 	"github.com/FAU-CDI/wisski-distillery/core"
-	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/embed"
 	"github.com/FAU-CDI/wisski-distillery/internal/execx"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
 	"github.com/FAU-CDI/wisski-distillery/internal/stack"
@@ -143,7 +143,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
 	}
 
 	if err := logging.LogOperation(func() error {
-		return distillery.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), func(dst, src string) {
+		return embed.InstallResource(dis.RuntimeDir(), filepath.Join("resources", "runtime"), func(dst, src string) {
 			context.Printf("[copy]  %s\n", dst)
 		})
 	}, context.IOStream, "Unpacking Runtime Components"); err != nil {

cmd/wdcli/main.go

@@ -51,6 +51,7 @@ func init() {
 
 	// servers
 	wdcli.Register(cmd.DisServer)
+	wdcli.Register(cmd.ResolverServer)
 }
 
 // an error when no arguments are provided.

distillery/resources/compose/reserve/.env.sample

@@ -1,14 +0,0 @@
-#######################
-# Meta Settings
-#######################
-
-#######################
-### Web Server settings
-#######################
-# the hostname for the website
-VIRTUAL_HOST=example.com
-
-# optional letsencrypt support
-# when blank, ignore
-LETSENCRYPT_HOST=
-LETSENCRYPT_EMAIL=

distillery/resources/templates/docker-env/resolver

@@ -1,9 +0,0 @@
-VIRTUAL_HOST=${VIRTUAL_HOST}
-
-PREFIX_FILE=${PREFIX_FILE}
-
-DEFAULT_DOMAIN=${DEFAULT_DOMAIN}
-LEGACY_DOMAIN=${LEGACY_DOMAIN}
-
-LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
-LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}

embed/resources.go

@@ -1,5 +1,5 @@
-// TODO: Rename this to resources oncen finished
+// Package embed contains embedded resources
-package distillery
+package embed
 
 import (
 	"embed"

embed/resources/compose/resolver/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.io/library/alpine
+
+COPY wdcli /wdcli
+EXPOSE 8888
+CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]

embed/resources/compose/resolver/docker-compose.yml

@@ -2,26 +2,26 @@ version: "3.7"
 
 services:
   wdresolve:
-    image: ghcr.io/fau-cdi/wdresolve:latest
+    build: .
     restart: always
-    volumes:
-      - "${PREFIX_FILE}:/prefixes.cfg:ro"
     environment:
       # port and hostname for this image to use
       VIRTUAL_HOST: ${VIRTUAL_HOST}
-      VIRTUAL_PORT: 8080
+      VIRTUAL_PORT: 8888
       VIRTUAL_PATH: /go/
+
+      CONFIG_PATH: ${CONFIG_PATH}
       
       # optional letsencrypt email
       LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
       LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
 
-      # default and legacy domain
+    volumes:
-      DEFAULT_DOMAIN: ${DEFAULT_DOMAIN}
+      - "${CONFIG_PATH}:${CONFIG_PATH}:ro"
-      LEGACY_DOMAIN: ${LEGACY_DOMAIN}
+      - "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
-
+      - "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
-      # the prefix file
+      - "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
-      PREFIX_FILE: /prefixes.cfg
+      - "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
 
 networks:
   default:

embed/resources/templates/docker-env/resolver

@@ -0,0 +1,10 @@
+VIRTUAL_HOST=${VIRTUAL_HOST}
+
+LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
+LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
+
+CONFIG_PATH=${CONFIG_PATH}
+DEPLOY_ROOT=${DEPLOY_ROOT}
+GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
+SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
+RESOLVER_CONFIG=${RESOLVER_CONFIG}

embed/resources_template.go

@@ -1,4 +1,4 @@
-package distillery
+package embed
 
 import (
 	"io"

env/component.go

@@ -34,12 +34,15 @@ type Component interface {
 
 // asCoreStack treats the provided stack as a core component of this distillery.
 func (dis *Distillery) makeComponentStack(component Component, stack stack.Installable) stack.Installable {
+	stack.Dir = dis.getComponentPath(component)
+
 	name := component.Name()
-
-	stack.Dir = filepath.Join(dis.Config.DeployRoot, "core", name)
-
 	stack.ContextResource = filepath.Join("resources", "compose", name)
 	stack.EnvFileResource = filepath.Join("resources", "templates", "docker-env", name)
 
 	return stack
 }
+
+func (dis *Distillery) getComponentPath(component Component) string {
+	return filepath.Join(dis.Config.DeployRoot, "core", component.Name())
+}

env/component_resolver.go

@@ -1,10 +1,16 @@
 package env
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
-	"strings"
+	"regexp"
 
+	"github.com/FAU-CDI/wdresolve"
+	"github.com/FAU-CDI/wdresolve/resolvers"
+	"github.com/FAU-CDI/wisski-distillery/core"
 	"github.com/FAU-CDI/wisski-distillery/internal/stack"
+	"github.com/tkw1536/goprogram/stream"
 )
 
 // ResolverComponent represents the 'resolver' layer belonging to a distillery
@@ -28,28 +34,76 @@ func (ResolverComponent) Name() string {
 }
 
 func (resolver ResolverComponent) Stack() stack.Installable {
-	stack := resolver.dis.makeComponentStack(resolver, stack.Installable{
+	return resolver.dis.makeComponentStack(resolver, stack.Installable{
 		EnvFileContext: map[string]string{
 			"VIRTUAL_HOST":      resolver.dis.DefaultVirtualHost(),
 			"LETSENCRYPT_HOST":  resolver.dis.DefaultLetsencryptHost(),
 			"LETSENCRYPT_EMAIL": resolver.dis.Config.CertbotEmail,
-			"PREFIX_FILE":       "", // set below!
-			"DEFAULT_DOMAIN":    resolver.dis.Config.DefaultDomain,
-			"LEGACY_DOMAIN":     strings.Join(resolver.dis.Config.SelfExtraDomains, ","),
-		},
 
-		TouchFiles: []string{resolver.ConfigName},
+			"CONFIG_PATH": resolver.dis.Config.ConfigPath,
+			"DEPLOY_ROOT": resolver.dis.Config.DeployRoot,
+
+			"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.dis.Config.GlobalAuthorizedKeysFile,
+			"SELF_OVERRIDES_FILE":         resolver.dis.Config.SelfOverridesFile,
+			"RESOLVER_CONFIG":             resolver.ConfigPath(),
+		},
+		CopyContextFiles: []string{core.Executable},
 	})
-	stack.EnvFileContext["PREFIX_FILE"] = filepath.Join(stack.Dir, resolver.ConfigName)
-	return stack
 }
 
-func (ResolverComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
+func (resolver ResolverComponent) Context(parent stack.InstallationContext) stack.InstallationContext {
-	return parent
+	return stack.InstallationContext{
+		core.Executable: resolver.dis.CurrentExecutable(),
+	}
+}
+
+func (resolver ResolverComponent) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
+	p.TrustXForwardedProto = true
+
+	fallback := &resolvers.Regexp{
+		Data: map[string]string{},
+	}
+
+	// handle the default domain name!
+	domainName := resolver.dis.Config.DefaultDomain
+	if domainName != "" {
+		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
+		io.Printf("registering default domain %s\n", domainName)
+	}
+
+	// handle the extra domains!
+	for _, domain := range resolver.dis.Config.SelfExtraDomains {
+		fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
+		io.Printf("registering legacy domain %s\n", domain)
+	}
+
+	// open the prefix file
+	prefixFile := resolver.ConfigPath()
+	fs, err := os.Open(prefixFile)
+	io.Println("loading prefixes from ", prefixFile)
+	if err != nil {
+		return p, err
+	}
+	defer fs.Close()
+
+	// read the prefixes
+	// TODO: Do we want to load these without a file?
+	prefixes, err := resolvers.ReadPrefixes(fs)
+	if err != nil {
+		return p, err
+	}
+
+	// and use that as the resolver!
+	p.Resolver = resolvers.InOrder{
+		prefixes,
+		fallback,
+	}
+
+	return p, nil
 }
 
 func (resolver ResolverComponent) Path() string {
-	return resolver.Stack().Dir
+	return resolver.dis.getComponentPath(resolver)
 }
 
 func (resolver ResolverComponent) ConfigPath() string {

env/component_triplestore.go

@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/embed"
 	"github.com/FAU-CDI/wisski-distillery/internal/logging"
 	"github.com/FAU-CDI/wisski-distillery/internal/stack"
 	"github.com/FAU-CDI/wisski-distillery/internal/wait"
@@ -154,7 +154,7 @@ func (ts TriplestoreComponent) Provision(name, domain, user, password string) er
 	}
 
 	// prepare the create repo request
-	createRepo, err := distillery.ReadTemplate(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), map[string]string{
+	createRepo, err := embed.ReadTemplate(filepath.Join("resources", "templates", "repository", "graphdb-repo.ttl"), map[string]string{
 		"GRAPHDB_REPO":    name,
 		"INSTANCE_DOMAIN": domain,
 	})

go.mod

@@ -14,6 +14,7 @@ require (
 )
 
 require (
+	github.com/FAU-CDI/wdresolve v0.0.0-20220909150742-34bde844301d // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/FAU-CDI/wdresolve v0.0.0-20220909150742-34bde844301d h1:boaE5bElltxs75SrC/sCjmsPmMnk0kcSuMYCjeo6pfE=
+github.com/FAU-CDI/wdresolve v0.0.0-20220909150742-34bde844301d/go.mod h1:9L2AZlOgDtg1WZ9Z1P1nBjqOY/b0mVrVE7UvO20pO2Q=
 github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM=
 github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko=
 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=

internal/stack/installable.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/FAU-CDI/wisski-distillery/distillery"
+	"github.com/FAU-CDI/wisski-distillery/embed"
 	"github.com/FAU-CDI/wisski-distillery/internal/fsx"
 	"github.com/pkg/errors"
 	"github.com/tkw1536/goprogram/stream"
@@ -38,7 +38,7 @@ type InstallationContext map[string]string
 // InstallationContext
 func (is Installable) Install(io stream.IOStream, context InstallationContext) error {
 	// setup the base files
-	if err := distillery.InstallResource(
+	if err := embed.InstallResource(
 		is.Dir,
 		is.ContextResource,
 		func(dst, src string) {
@@ -52,7 +52,7 @@ func (is Installable) Install(io stream.IOStream, context InstallationContext) e
 	envDest := filepath.Join(is.Dir, ".env")
 	if is.EnvFileResource != "" && is.EnvFileContext != nil {
 		io.Printf("[config]  %s\n", envDest)
-		if err := distillery.InstallTemplate(
+		if err := embed.InstallTemplate(
 			envDest,
 			is.EnvFileResource,
 			is.EnvFileContext,