Merge all the server components

This commit is contained in:
Tom Wiesing 2022-09-15 15:04:35 +02:00
parent 85b5603d9d
commit f5f2ac1a03
No known key found for this signature in database
25 changed files with 365 additions and 352 deletions

View file

@ -4,11 +4,16 @@ import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
type Dis struct {
component.ComponentBase
Instances *instances.Instances
ResolverFile string
}
func (dis Dis) Name() string {
@ -35,7 +40,9 @@ func (dis Dis) Stack() component.Installable {
"GLOBAL_AUTHORIZED_KEYS_FILE": dis.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": dis.Config.SelfOverridesFile,
},
CopyContextFiles: []string{dis.Config.CurrentExecutable()},
TouchFiles: []string{dis.ResolverFile},
CopyContextFiles: []string{core.Executable},
})
}

View file

@ -0,0 +1,26 @@
package dis
import (
"net/http"
"github.com/tkw1536/goprogram/stream"
)
func (dis Dis) info(io stream.IOStream) (http.Handler, error) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
all, err := dis.Instances.All()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("internal server error"))
io.EPrintln(err)
return
}
for _, wk := range all {
w.WriteHeader(http.StatusOK)
w.Write([]byte(wk.Slug))
w.Write([]byte("\n"))
}
}), nil
}

View file

@ -0,0 +1,61 @@
package dis
import (
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/tkw1536/goprogram/stream"
)
func (dis Dis) ResolverConfigPath() string {
return filepath.Join(dis.Dir, dis.ResolverFile)
}
func (dis Dis) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := dis.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range dis.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := dis.ResolverConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}

View file

@ -0,0 +1,137 @@
package dis
import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"github.com/tkw1536/goprogram/stream"
)
// self returns the handler for the self overrides
func (dis Dis) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(dis.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", dis.Config.SelfOverridesFile)
if err != nil {
return redirect, err
}
defer overrides.Close()
// decode the overrides file
if err := json.NewDecoder(overrides).Decode(&redirect.Overrides); err != nil {
return redirect, err
}
if redirect.Overrides == nil {
redirect.Overrides = make(map[string]string)
}
redirect.Overrides["/"] = dis.Config.SelfRedirect.String()
// create a redirect server
redirect.Fallback, err = dis.selfFallback()
if err != nil {
return redirect, err
}
redirect.Absolute = false
redirect.Overrides = nil
redirect.Permanent = false
// and return!
return redirect, nil
}
func (dis *Dis) selfFallback() (http.Handler, error) {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
dis.serveFallback(w, r)
}), nil
}
var notFoundText = []byte("not found")
func (dis *Dis) serveFallback(w http.ResponseWriter, r *http.Request) {
slug := dis.Config.SlugFromHost(r.Host)
if slug == "" {
w.WriteHeader(http.StatusNotFound)
w.Write(notFoundText)
return
}
if ok, _ := dis.Instances.Has(slug); !ok {
w.WriteHeader(http.StatusNotFound)
fmt.Fprintf(w, "WissKI %q not found\n", slug)
return
}
w.WriteHeader(http.StatusBadGateway)
fmt.Fprintf(w, "WissKI %q is currently offline\n", slug)
}
// Redirect implements a redirect server that redirects all requests.
// It implements http.Handler.
type Redirect struct {
// Target is the target URL to redirect to.
Target string
// Fallback is used when target is the empty string.
Fallback http.Handler
// Absolute determines if the request path should be appended to the target URL when redirecting.
// By default this path is always appended, set Absolute to true to prevent this.
Absolute bool
// Overrides is a map from paths to URLs that should override the default target.
Overrides map[string]string
// Permanent determines if the redirect responses issued should return
// Permanent Redirect (Status Code 308) or Temporary Redirect (Status Code 307).
Permanent bool
}
// Redirect determines the redirect URL for a specific incoming request
// If it returns the empty string, the fallback is used.
func (redirect Redirect) Redirect(r *http.Request) string {
// if we have an override for this URL, use it immediatly
url := strings.TrimSuffix(r.URL.Path, "/")
if override, ok := redirect.Overrides[url]; ok {
return override
}
if redirect.Target == "" {
return ""
}
// if we are in absolute redirect mode, always return the absolute URL
if redirect.Absolute {
return redirect.Target
}
// return the target + the redirected URL
dest := strings.TrimSuffix(redirect.Target, "/") + r.URL.Path
if len(r.URL.RawQuery) > 0 {
dest += "?" + r.URL.RawQuery
}
return dest
}
// ServeHTTP implements the http.Handler interface and redirects a single request to redirect.Target.
func (redirect Redirect) ServeHTTP(w http.ResponseWriter, r *http.Request) {
dest := redirect.Redirect(r)
if dest == "" {
redirect.Fallback.ServeHTTP(w, r)
return
}
// determine if we are temporary or permanent redirect
status := http.StatusTemporaryRedirect
if redirect.Permanent {
status = http.StatusPermanentRedirect
}
// and do the redirect
http.Redirect(w, r, dest, status)
}

View file

@ -0,0 +1,39 @@
package dis
import (
"net/http"
"github.com/tkw1536/goprogram/stream"
)
// Server returns an http.Mux that implements the main server instance
func (dis Dis) Server(io stream.IOStream) (http.Handler, error) {
// self server
self, err := dis.self(io)
if err != nil {
return nil, err
}
resolver, err := dis.resolver(io)
if err != nil {
return nil, err
}
info, err := dis.info(io)
if err != nil {
return nil, err
}
// resolver
mux := http.NewServeMux()
mux.Handle("/", self)
mux.Handle("/go/", resolver)
mux.Handle("/wisski/navigate", resolver)
// TODO: Fix me!
mux.Handle("/dis/", info)
return mux, nil
}

View file

@ -2,4 +2,4 @@ FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","dis_server","--bind","0.0.0.0:8888"]
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","server","--bind","0.0.0.0:8888"]

View file

@ -8,7 +8,6 @@ services:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /dis/
CONFIG_PATH: ${CONFIG_PATH}

View file

@ -90,7 +90,7 @@ func (is Installable) Install(io stream.IOStream, context InstallationContext) e
// find the source!
src, ok := context[name]
if !ok {
return errors.Errorf("Missing file from context: %s", src)
return errors.Errorf("Missing file from context: %q", src)
}
// find the destination!

View file

@ -18,14 +18,16 @@ var errInvalidSlug = errors.New("not a valid slug")
//
// It does not perform any checks if the instance already exists, or does the creation in the database.
func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
wisski.instances = instances
// make sure that the slug is valid!
if _, err := stringparser.ParseSlug(slug); err != nil {
slug, err = stringparser.ParseSlug(slug)
if err != nil {
return wisski, errInvalidSlug
}
wisski.Instance.Slug = slug
wisski.Instance.FilesystemBase = filepath.Join(instances.Dir, slug)
wisski.Instance.FilesystemBase = filepath.Join(instances.Dir, wisski.Domain())
wisski.Instance.OwnerEmail = ""
wisski.Instance.AutoBlindUpdateEnabled = true
@ -60,7 +62,6 @@ func (instances *Instances) Create(slug string) (wisski WissKI, err error) {
}
// store the instance in the object and return it!
wisski.instances = instances
return wisski, nil
}

View file

@ -1,10 +0,0 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}
RESOLVER_CONFIG=${RESOLVER_CONFIG}

View file

@ -1,109 +0,0 @@
package resolver
import (
"embed"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/tkw1536/goprogram/stream"
)
// TODO: Add a 'self-server' concept!
type Resolver struct {
component.ComponentBase
ConfigName string // the name to the config file
Executable string // path to the current executable
}
func (Resolver) Name() string {
return "resolver"
}
func (resolver Resolver) ConfigPath() string {
return filepath.Join(resolver.Dir, resolver.ConfigName)
}
//go:embed all:stack resolver.env
var resources embed.FS
func (resolver Resolver) Stack() component.Installable {
return resolver.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "resolver.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": resolver.Config.DefaultHost(),
"LETSENCRYPT_HOST": resolver.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": resolver.Config.CertbotEmail,
"CONFIG_PATH": resolver.Config.ConfigPath,
"DEPLOY_ROOT": resolver.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": resolver.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": resolver.Config.SelfOverridesFile,
"RESOLVER_CONFIG": resolver.ConfigPath(),
},
TouchFiles: []string{resolver.ConfigName},
CopyContextFiles: []string{core.Executable},
})
}
func (resolver Resolver) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: resolver.Executable,
}
}
func (resolver Resolver) Server(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := resolver.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range resolver.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := resolver.ConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}

View file

@ -1,5 +0,0 @@
FROM docker.io/library/alpine
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","--config","${CONFIG_PATH}","resolver_server","--bind","0.0.0.0:8888"]

View file

@ -1,29 +0,0 @@
version: "3.7"
services:
wdresolve:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
VIRTUAL_PATH: /go/
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
- "${RESOLVER_CONFIG}:${RESOLVER_CONFIG}:ro"
networks:
default:
name: distillery
external: true

View file

@ -1,7 +0,0 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
TARGET=${TARGET}
OVERRIDES_FILE=${OVERRIDES_FILE}

View file

@ -1,42 +0,0 @@
package self
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
)
type Self struct {
component.ComponentBase
}
func (Self) Name() string {
return "self"
}
//go:embed all:stack
//go:embed self.env
var resources embed.FS
func (self Self) Stack() component.Installable {
// TODO: Move me into config!
TARGET := "https://github.com/FAU-CDI/wisski-distillery"
if self.Config.SelfRedirect != nil { // TODO: move to config!
TARGET = self.Config.SelfRedirect.String()
}
return self.ComponentBase.MakeStack(component.Installable{
Resources: resources,
ContextPath: "stack",
EnvPath: "self.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": self.Config.DefaultHost(),
"LETSENCRYPT_HOST": self.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": self.Config.CertbotEmail,
"TARGET": TARGET,
"OVERRIDES_FILE": self.Config.SelfOverridesFile,
},
})
}

View file

@ -1,28 +0,0 @@
version: "3.7"
services:
tr:
image: ghcr.io/tkw1536/tr:latest
restart: always
volumes:
- "${OVERRIDES_FILE}:/overrides.json:ro"
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8080
VIRTUAL_PATH: /
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
# the overrides file
OVERRIDES: /overrides.json
# where to redirect to
TARGET: ${TARGET}
networks:
default:
name: distillery
external: true

View file

@ -26,7 +26,7 @@ type Config struct {
// By default, the default domain redirects to the distillery repository.
// If you want to change this, set an alternate domain name here.
SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"" parser:"https_url"`
SelfRedirect *url.URL `env:"SELF_REDIRECT" default:"https://github.com/FAU-CDI/wisski-distillery" parser:"https_url"`
// By default, only the 'self' domain above is caught.
// To catch additional domains, add them here (comma seperated)

View file

@ -39,3 +39,19 @@ func (cfg Config) DefaultHost() string {
func (cfg Config) DefaultSSLHost() string {
return cfg.IfHttps(cfg.DefaultHost())
}
// SlugFromHost returns the slug belonging to the appropriate host.
func (cfg Config) SlugFromHost(host string) (slug string) {
// extract an ':port' that happens to be in the host.
domain, _, _ := strings.Cut(host, ":")
// check all the possible domain endings
for _, suffix := range append([]string{cfg.DefaultDomain}, cfg.SelfExtraDomains...) {
if strings.HasSuffix(domain, "."+suffix) {
return domain[:len(domain)-len(suffix)-1]
}
}
// no domain found!
return ""
}

View file

@ -8,13 +8,12 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/dis"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/component/resolver"
"github.com/FAU-CDI/wisski-distillery/internal/component/self"
"github.com/FAU-CDI/wisski-distillery/internal/component/sql"
"github.com/FAU-CDI/wisski-distillery/internal/component/ssh"
"github.com/FAU-CDI/wisski-distillery/internal/component/triplestore"
"github.com/FAU-CDI/wisski-distillery/internal/component/web"
"github.com/FAU-CDI/wisski-distillery/internal/core"
"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
)
// components holds the various components of the distillery
@ -24,16 +23,14 @@ import (
type components struct {
// installable components
web *web.Web
self *self.Self
resolver *resolver.Resolver
dis *dis.Dis
ssh *ssh.SSH
ts *triplestore.Triplestore
sql *sql.SQL
web lazy.Lazy[*web.Web]
dis lazy.Lazy[*dis.Dis]
ssh lazy.Lazy[*ssh.SSH]
ts lazy.Lazy[*triplestore.Triplestore]
sql lazy.Lazy[*sql.SQL]
// other components
instances *instances.Instances
instances lazy.Lazy[*instances.Instances]
}
// makeComponent makes or returns a component inside the [component] struct of the distillery
@ -45,7 +42,7 @@ type components struct {
// init is called with a new non-nil component to initialize it. It may be nil, to indicate no initialization is required.
//
// makeComponent returns the new or existing component instance
func makeComponent[C component.Component](dis *Distillery, field *C, init func(C)) C {
func makeComponent[C component.Component](dis *Distillery, field *lazy.Lazy[C], init func(C)) C {
// get the typeof C and make sure that it is a pointer type!
typC := reflect.TypeOf((*C)(nil)).Elem()
@ -53,32 +50,27 @@ func makeComponent[C component.Component](dis *Distillery, field *C, init func(C
panic("makeComponent: C must be backed by a pointer")
}
// if the component is non-nil, then it has already been initialized
if !reflect.ValueOf(*field).IsNil() {
return *field
}
// return the field
return field.Get(func() (c C) {
c = reflect.New(typC.Elem()).Interface().(C)
if init != nil {
init(c)
}
// create a new element, and call the initializer (if requested)
*field = reflect.New(typC.Elem()).Interface().(C)
if init != nil {
init(*field)
}
base := c.Base()
base.Config = dis.Config
if base.Dir == "" {
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", c.Name())
}
// apply the base configuration
base := (*field).Base()
base.Config = dis.Config
base.Dir = filepath.Join(dis.Config.DeployRoot, "core", (*field).Name())
// and eventually return it
return *field
return
})
}
// Components returns all components that have a stack function
func (dis *Distillery) Components() []component.InstallableComponent {
return []component.InstallableComponent{
dis.Web(),
dis.Self(),
dis.Resolver(),
dis.Dis(),
dis.SSH(),
dis.Triplestore(),
@ -90,18 +82,11 @@ func (dis *Distillery) Web() *web.Web {
return makeComponent(dis, &dis.components.web, nil)
}
func (dis *Distillery) Self() *self.Self {
return makeComponent(dis, &dis.components.self, nil)
}
func (dis *Distillery) Resolver() *resolver.Resolver {
return makeComponent(dis, &dis.components.resolver, func(resolver *resolver.Resolver) {
resolver.ConfigName = core.PrefixConfig
})
}
func (d *Distillery) Dis() *dis.Dis {
return makeComponent(d, &d.components.dis, nil)
return makeComponent(d, &d.components.dis, func(ddis *dis.Dis) {
ddis.ResolverFile = core.PrefixConfig
ddis.Instances = d.Instances()
})
}
func (dis *Distillery) SSH() *ssh.SSH {
@ -126,6 +111,7 @@ func (dis *Distillery) Triplestore() *triplestore.Triplestore {
func (dis *Distillery) Instances() *instances.Instances {
return makeComponent(dis, &dis.components.instances, func(instances *instances.Instances) {
instances.Dir = filepath.Join(dis.Config.DeployRoot, "instances")
instances.SQL = dis.SQL()
instances.TS = dis.Triplestore()
})

View file

@ -1,33 +0,0 @@
package wisski
import (
"io"
"net/http"
)
// TODO: Move this into dis!
// Server represents a server for this distillery
type Server struct {
dis *Distillery
}
func (dis *Distillery) Server() *Server {
return &Server{
dis: dis,
}
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
instances, err := s.dis.Instances().All()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
io.WriteString(w, "Something went wrong")
return
}
w.WriteHeader(http.StatusOK)
for _, instance := range instances {
io.WriteString(w, instance.Slug+"\n")
}
}