'wdcli backup': Rework backup process

This commit reworks the backup process to dynamically find the list of
components.
This commit is contained in:
Tom Wiesing 2022-09-17 16:30:32 +02:00
parent 55bee7422d
commit 5cd5ae9be2
No known key found for this signature in database
32 changed files with 361 additions and 279 deletions

View file

@ -0,0 +1,48 @@
package control
import (
"io/fs"
"os"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/stream"
)
func (*Control) BackupName() string {
return "config"
}
// Backup backups all control plane configuration files into dest
func (control *Control) Backup(io stream.IOStream, dest string) error {
// create the destination directory, TODO: outsource this
if err := os.Mkdir(dest, fs.ModeDir); err != nil {
return err
}
files := control.backupFiles()
for _, src := range files {
dst := filepath.Join(dest, filepath.Base(src)) // destination path
// if the src file does not exist, don't copy it!
if !fsx.IsFile(src) { // TODO: log this somewhere
continue
}
if err := fsx.CopyFile(dst, src); err != nil {
return err
}
}
return nil
}
// backupfiles lists the files to be backed up.
func (control *Control) backupFiles() []string {
return []string{
control.Config.ConfigPath,
control.Config.ExecutablePath(),
control.Config.SelfOverridesFile,
control.Config.GlobalAuthorizedKeysFile,
}
}

View file

@ -0,0 +1,9 @@
VIRTUAL_HOST=${VIRTUAL_HOST}
LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
CONFIG_PATH=${CONFIG_PATH}
DEPLOY_ROOT=${DEPLOY_ROOT}
GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE}
SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE}

View file

@ -0,0 +1,54 @@
package control
import (
"embed"
"github.com/FAU-CDI/wisski-distillery/internal/component"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/core"
)
// Control represents the control server
type Control struct {
component.ComponentBase
Instances *instances.Instances
ResolverFile string
}
func (control Control) Name() string {
return "dis" // TODO: Rename this to control!
}
//go:embed all:control control.env
var resources embed.FS
func (control Control) Stack() component.StackWithResources {
return control.ComponentBase.MakeStack(component.StackWithResources{
Resources: resources,
ContextPath: "control",
EnvPath: "control.env",
EnvContext: map[string]string{
"VIRTUAL_HOST": control.Config.DefaultHost(),
"LETSENCRYPT_HOST": control.Config.DefaultSSLHost(),
"LETSENCRYPT_EMAIL": control.Config.CertbotEmail,
"CONFIG_PATH": control.Config.ConfigPath,
"DEPLOY_ROOT": control.Config.DeployRoot,
"GLOBAL_AUTHORIZED_KEYS_FILE": control.Config.GlobalAuthorizedKeysFile,
"SELF_OVERRIDES_FILE": control.Config.SelfOverridesFile,
},
TouchFiles: []string{control.ResolverFile},
CopyContextFiles: []string{core.Executable},
})
}
func (control Control) Context(parent component.InstallationContext) component.InstallationContext {
return component.InstallationContext{
core.Executable: control.Config.CurrentExecutable(),
}
}

View file

@ -0,0 +1,5 @@
FROM docker.io/library/docker:20.10-cli
COPY wdcli /wdcli
EXPOSE 8888
CMD ["/wdcli","--internal-in-docker","server","--bind","0.0.0.0:8888"]

View file

@ -0,0 +1,29 @@
version: "3.7"
services:
dis:
build: .
restart: always
environment:
# port and hostname for this image to use
VIRTUAL_HOST: ${VIRTUAL_HOST}
VIRTUAL_PORT: 8888
CONFIG_PATH: ${CONFIG_PATH}
# optional letsencrypt email
LETSENCRYPT_HOST: ${LETSENCRYPT_HOST}
LETSENCRYPT_EMAIL: ${LETSENCRYPT_EMAIL}
volumes:
# TODO: Mount docker socket properly!
- "/var/run/docker.sock:/var/run/docker.sock"
- "${CONFIG_PATH}:${CONFIG_PATH}:ro"
- "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro"
- "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro"
- "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro"
networks:
default:
name: distillery
external: true

View file

@ -0,0 +1,55 @@
<!DOCTYPE html>
<link rel="stylesheet" href="/dis/static/dis.css">
<link rel="stylesheet" href="/dis/static/autolink.css">
<title>Distillery Status Page</title>
<h1 id="top">Distillery Status Page</h1>
<h2 id="overview">Overview</h2>
<p>
<b>Domain:</b> <code>{{.Config.DefaultDomain}}</code> <br />
<b>Legacy Domain(s):</b> <code>{{.Config.SelfExtraDomains}}</code><br />
<b>HTTPS Email:</b> <code>{{.Config.CertbotEmail}}</code><br />
<hr />
<b>Homepage Redirect:</b><a href="{{.Config.SelfRedirect}}" target="_blank" rel="noopener noreferrer">{{.Config.SelfRedirect}}</a><br />
<hr />
<b>Backup Age:</b> <code>{{.Config.MaxBackupAge}}</code> Day(s)<br />
<hr />
<b>Base Directory:</b> <code>{{.Config.DeployRoot}}</code><br />
<b>Configuration File:</b> <code>{{.Config.ConfigPath}}</code><br />
<b>Authorized_Keys File:</b> <code>{{.Config.GlobalAuthorizedKeysFile}}</code><br />
<hr />
<b>MySQL User Prefix:</b> <code>{{.Config.MysqlUserPrefix}}</code><br />
<b>MySQL Database Prefix:</b> <code>{{.Config.MysqlDatabasePrefix}}</code><br />
<b>GraphDB User Prefix:</b> <code>{{.Config.GraphDBUserPrefix}}</code><br />
<b>GraphDB Database Prefix:</b> <code>{{.Config.GraphDBRepoPrefix}}</code><br />
<hr />
<b>Bookkeeping Database:</b> <code>{{.Config.DistilleryBookkeepingDatabase}}</code><br />
<b>Bookkeeping Table:</b> <code>{{.Config.DistilleryBookkeepingTable}}</code><br />
</p>
<h2 id="instances">Instances</h2>
<p>
<code>{{ .TotalCount }}</code> instance(s) = <code>{{ .RunningCount }}</code> running + <code>{{ .StoppedCount }}</code> stopped<br />
</p>
{{range .Instances}}
<div class="wisski {{ if .Running }}running{{ else }}stopped{{ end }}">
<h3 id="instance-{{.Slug}}">{{.Slug}}{{ if not .Running }}&nbsp;<small>not running</small>{{ end }}</h3>
<p>
<a href="{{.URL}}" target="_blank" rel="noopener noreferrer">{{.URL}}</a><br />
<small>
<a href="/dis/instance/{{.Slug}}">More Details</a>
</small>
</p>
</div>
{{end}}
<footer>
Generated at <code>{{ .Time }}</code>
</footer>
<script src="/dis/static/autolink.js"></script>

View file

@ -0,0 +1,36 @@
<!DOCTYPE html>
<link rel="stylesheet" href="/dis/static/dis.css">
<link rel="stylesheet" href="/dis/static/autolink.css">
<title>Distillery Status Page - {{ .Info.Slug }}</title>
<h1 id="top">Distillery Status Page - {{ .Info.Slug }}</h1>
<p>
<a href="/dis/index">Back to index</a>
</p>
<p>
<b>Slug:</b> <code>{{ .Info.Slug }}</code> <br />
<b>URL:</b> <a href="{{ .Info.URL }}" target="_blank" rel="noopener noreferrer">{{ .Info.URL }}</a> <br />
<hr />
<b>Running:</b> <code>{{ .Info.Running }}</code> <br />
<hr />
<b>Created:</b> <code>{{ .Instance.Created }}</code> <br />
<b>OwnerEmail:</b> <code>{{ .Instance.OwnerEmail }}</code> <br />
<hr />
<b>FilesystemBase:</b> <code>{{ .Instance.FilesystemBase }}</code> <br />
<b>AutoBlindUpdateEnabled:</b> <code>{{ .Instance.AutoBlindUpdateEnabled }}</code> <br />
<hr />
<b>Pathbuilders:</b> <code>{{ .Info.Pathbuilders }}</code> <br />
<hr />
<b>SqlDatabase:</b> <code>{{ .Instance.SqlDatabase }}</code> <br />
<b>SqlUsername:</b> <code>{{ .Instance.SqlUsername }}</code> <br />
<hr />
<b>GraphDBRepository:</b> <code>{{ .Instance.GraphDBRepository }}</code> <br />
<b>GraphDBUsername:</b> <code>{{ .Instance.GraphDBUsername }}</code> <br />
</p>
<footer>
Generated at <code>{{ .Time }}</code>
</footer>
<script src="/dis/static/autolink.js"></script>

View file

@ -0,0 +1,20 @@
.header-link {
position: relative;
left: 0.5em;
opacity: 0;
font-size: 0.8em;
-webkit-transition: opacity 0.2s ease-in-out 0.1s;
-moz-transition: opacity 0.2s ease-in-out 0.1s;
-ms-transition: opacity 0.2s ease-in-out 0.1s;
text-decoration: none;
}
h2:hover .header-link,
h3:hover .header-link,
h4:hover .header-link,
h5:hover .header-link,
h6:hover .header-link {
opacity: 1;
}

View file

@ -0,0 +1,24 @@
/** adding links to each item, see http://blog.parkermoore.de/2014/08/01/header-anchor-links-in-vanilla-javascript-for-github-pages-and-jekyll/ */
var anchorForId = function (id) {
var anchor = document.createElement("a");
anchor.className = "header-link";
anchor.href = "#" + id;
anchor.innerHTML = "#";
return anchor;
};
var linkifyAnchors = function (level) {
var headers = document.getElementsByTagName("h" + level);
for (var h = 0; h < headers.length; h++) {
var header = headers[h];
if (typeof header.id !== "undefined" && header.id !== "") {
header.appendChild(anchorForId(header.id));
}
}
};
for (var level = 1; level <= 6; level++) {
linkifyAnchors(level);
}

View file

@ -0,0 +1,67 @@
html {
line-height: 1.5;
font-family: Roboto;
font-size: 20px;
color: #1a1a1a;
background-color: #fdfdfd;
}
body {
margin: 0 auto;
max-width: 36em;
padding-left: 50px;
padding-right: 50px;
padding-top: 50px;
padding-bottom: 50px;
hyphens: auto;
overflow-wrap: break-word;
text-rendering: optimizeLegibility;
font-kerning: normal;
}
@media (max-width: 600px) {
body {
font-size: 0.9em;
padding: 1em;
}
}
h1 {
margin-top: 1.4em;
}
h2,h3 {
margin-top: 1em;
}
code {
font-family: Roboto Mono;
color: blue;
}
p {
margin: 1em 0;
text-align: justify;
}
a, a:visited {
color: #1a1a1a;
}
footer {
border-top: 1px solid #1a1a1a;
font-size: small;
text-align: center;
}
.wisski {
padding-left: 5px;
}
.wisski.running {
background-color: green;
}
.wisski.stopped {
background-color: red;
}

View file

@ -0,0 +1,196 @@
package control
import (
"embed"
"html/template"
"io/fs"
"net/http"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/bookkeeping"
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/config"
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
)
func (control *Control) info(io stream.IOStream) (http.Handler, error) {
mux := http.NewServeMux()
// handle everything under /dis/!
mux.HandleFunc("/dis/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/dis/" {
http.Redirect(w, r, "/dis/index", http.StatusTemporaryRedirect)
return
}
http.NotFound(w, r)
})
// static stuff
static, err := control.disStatic()
if err != nil {
return nil, err
}
mux.Handle("/dis/static/", static)
// render everything
mux.Handle("/dis/index", httpx.HTMLHandler[disIndex]{
Handler: control.disIndex,
Template: indexTemplate,
})
mux.Handle("/dis/instance/", httpx.HTMLHandler[disInstance]{
Handler: control.disInstance,
Template: instanceTemplate,
})
// api -- for future usage
mux.Handle("/dis/api/v1/instance/get/", httpx.JSON(control.getinstance))
mux.Handle("/dis/api/v1/instance/all", httpx.JSON(control.allinstances))
// ensure that everyone is logged in!
return httpx.BasicAuth(mux, "WissKI Distillery Admin", func(user, pass string) bool {
return user == control.Config.DisAdminUser && pass == control.Config.DisAdminPassword
}), nil
}
// disIndex is the context of the "/dis/index" page
type disIndex struct {
Time time.Time
Config *config.Config
Instances []instances.Info
TotalCount int
RunningCount int
StoppedCount int
}
func (dis *Control) disIndex(r *http.Request) (idx disIndex, err error) {
// load instances
idx.Instances, err = dis.allinstances(r)
if err != nil {
return
}
// count how many are running and how many are stopped
for _, i := range idx.Instances {
if i.Running {
idx.RunningCount++
} else {
idx.StoppedCount++
}
}
idx.TotalCount = len(idx.Instances)
// get the static properties
idx.Config = dis.Config
// current time
idx.Time = time.Now()
return
}
// disInstance is the context of the "/dis/instance/*" page
type disInstance struct {
Time time.Time
Instance bookkeeping.Instance
Info instances.Info
}
func (dis *Control) disInstance(r *http.Request) (is disInstance, err error) {
// find the slug as the last component of path!
slug := strings.TrimSuffix(r.URL.Path, "/")
slug = slug[strings.LastIndex(slug, "/")+1:]
// find the instance itself!
instance, err := dis.Instances.WissKI(slug)
if err == instances.ErrWissKINotFound {
return is, httpx.ErrNotFound
}
if err != nil {
return is, err
}
is.Instance = instance.Instance
// get some more info about the wisski
is.Info, err = instance.Info(false)
if err != nil {
return is, err
}
// current time
is.Time = time.Now()
return
}
//go:embed html/static
var htmlStaticFS embed.FS
func (*Control) disStatic() (http.Handler, error) {
fs, err := fs.Sub(htmlStaticFS, "html/static")
if err != nil {
return nil, err
}
return http.StripPrefix("/dis/static/", http.FileServer(http.FS(fs))), nil
}
//go:embed "html/index.html"
var indexTemplateStr string
var indexTemplate = template.Must(template.New("index.html").Parse(indexTemplateStr))
//go:embed "html/instance.html"
var instanceTemplateString string
var instanceTemplate = template.Must(template.New("instance.html").Parse(instanceTemplateString))
func (dis *Control) getinstance(r *http.Request) (info instances.Info, err error) {
// find the slug as the last component of path!
slug := strings.TrimSuffix(r.URL.Path, "/")
slug = slug[strings.LastIndex(slug, "/")+1:]
// load the wisski instance!
wisski, err := dis.Instances.WissKI(strings.TrimSuffix(slug, "/"))
if err == instances.ErrWissKINotFound {
return info, httpx.ErrNotFound
}
if err != nil {
return info, err
}
// get info about it!
return wisski.Info(false)
}
func (dis *Control) allinstances(*http.Request) (infos []instances.Info, err error) {
var errgroup errgroup.Group
// list all the instances
all, err := dis.Instances.All()
if err != nil {
return nil, err
}
// get all of their info!
infos = make([]instances.Info, len(all))
for i, instance := range all {
{
i := i
instance := instance
errgroup.Go(func() (err error) {
infos[i], err = instance.Info(true)
return err
})
}
}
// wait for the results, and return
err = errgroup.Wait()
return
}

View file

@ -0,0 +1,61 @@
package control
import (
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/FAU-CDI/wdresolve"
"github.com/FAU-CDI/wdresolve/resolvers"
"github.com/tkw1536/goprogram/stream"
)
func (control Control) ResolverConfigPath() string {
return filepath.Join(control.Dir, control.ResolverFile)
}
func (control Control) resolver(io stream.IOStream) (p wdresolve.ResolveHandler, err error) {
p.TrustXForwardedProto = true
fallback := &resolvers.Regexp{
Data: map[string]string{},
}
// handle the default domain name!
domainName := control.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range control.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
}
// open the prefix file
prefixFile := control.ResolverConfigPath()
fs, err := os.Open(prefixFile)
io.Println("loading prefixes from ", prefixFile)
if err != nil {
return p, err
}
defer fs.Close()
// read the prefixes
// TODO: Do we want to load these without a file?
prefixes, err := resolvers.ReadPrefixes(fs)
if err != nil {
return p, err
}
// and use that as the resolver!
p.Resolver = resolvers.InOrder{
prefixes,
fallback,
}
return p, nil
}

View file

@ -0,0 +1,134 @@
package control
import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"github.com/tkw1536/goprogram/stream"
)
// self returns the handler for the self overrides
func (control Control) self(io stream.IOStream) (redirect Redirect, err error) {
// open the overrides file
overrides, err := os.Open(control.Config.SelfOverridesFile)
io.Printf("loading overrides from %q\n", control.Config.SelfOverridesFile)
if err != nil {
return redirect, err
}
defer overrides.Close()
// decode the overrides file
if err := json.NewDecoder(overrides).Decode(&redirect.Overrides); err != nil {
return redirect, err
}
if redirect.Overrides == nil {
redirect.Overrides = make(map[string]string)
}
redirect.Overrides[""] = control.Config.SelfRedirect.String()
// create a redirect server
redirect.Fallback, err = control.selfFallback()
if err != nil {
return redirect, err
}
redirect.Absolute = false
redirect.Permanent = false
// and return!
return redirect, nil
}
func (control *Control) selfFallback() (http.Handler, error) {
return http.HandlerFunc(control.serveFallback), nil
}
var notFoundText = []byte("not found")
func (control *Control) serveFallback(w http.ResponseWriter, r *http.Request) {
slug := control.Config.SlugFromHost(r.Host)
if slug == "" {
w.WriteHeader(http.StatusNotFound)
w.Write(notFoundText)
return
}
if ok, _ := control.Instances.Has(slug); !ok {
w.WriteHeader(http.StatusNotFound)
fmt.Fprintf(w, "WissKI %q not found\n", slug)
return
}
w.WriteHeader(http.StatusBadGateway)
fmt.Fprintf(w, "WissKI %q is currently offline\n", slug)
}
// Redirect implements a redirect server that redirects all requests.
// It implements http.Handler.
type Redirect struct {
// Target is the target URL to redirect to.
Target string
// Fallback is used when target is the empty string.
Fallback http.Handler
// Absolute determines if the request path should be appended to the target URL when redirecting.
// By default this path is always appended, set Absolute to true to prevent this.
Absolute bool
// Overrides is a map from paths to URLs that should override the default target.
Overrides map[string]string
// Permanent determines if the redirect responses issued should return
// Permanent Redirect (Status Code 308) or Temporary Redirect (Status Code 307).
Permanent bool
}
// Redirect determines the redirect URL for a specific incoming request
// If it returns the empty string, the fallback is used.
func (redirect Redirect) Redirect(r *http.Request) string {
// if we have an override for this URL, use it immediatly
url := strings.TrimSuffix(r.URL.Path, "/")
if override, ok := redirect.Overrides[url]; ok {
return override
}
if redirect.Target == "" {
return ""
}
// if we are in absolute redirect mode, always return the absolute URL
if redirect.Absolute {
return redirect.Target
}
// return the target + the redirected URL
dest := strings.TrimSuffix(redirect.Target, "/") + r.URL.Path
if len(r.URL.RawQuery) > 0 {
dest += "?" + r.URL.RawQuery
}
return dest
}
// ServeHTTP implements the http.Handler interface and redirects a single request to redirect.Target.
func (redirect Redirect) ServeHTTP(w http.ResponseWriter, r *http.Request) {
dest := redirect.Redirect(r)
if dest == "" {
redirect.Fallback.ServeHTTP(w, r)
return
}
// determine if we are temporary or permanent redirect
status := http.StatusTemporaryRedirect
if redirect.Permanent {
status = http.StatusPermanentRedirect
}
// and do the redirect
http.Redirect(w, r, dest, status)
}

View file

@ -0,0 +1,39 @@
package control
import (
"net/http"
"github.com/tkw1536/goprogram/stream"
)
// Server returns an http.Mux that implements the main server instance
func (control Control) Server(io stream.IOStream) (http.Handler, error) {
// self server
self, err := control.self(io)
if err != nil {
return nil, err
}
resolver, err := control.resolver(io)
if err != nil {
return nil, err
}
info, err := control.info(io)
if err != nil {
return nil, err
}
// resolver
mux := http.NewServeMux()
mux.Handle("/", self)
mux.Handle("/go/", resolver)
mux.Handle("/wisski/get/", resolver)
// TODO: Fix me!
mux.Handle("/dis/", info)
return mux, nil
}