rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update to goprogram 0.1.0

Browse source
This commit is contained in:
Tom Wiesing 2022-10-06 13:38:29 +02:00
parent d2d681a4f2
commit 7cda92b342
No known key found for this signature in database
31 changed files with 141 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

3
internal/component/control/extras_info.go
View file

@ -1,6 +1,7 @@
package control
import (
"context"
"embed"
"html/template"
"io/fs"
@ -27,7 +28,7 @@ func (Info) Name() string { return "control-info" }
func (*Info) Routes() []string { return []string{"/dis/"} }
func (info *Info) Handler(route string, io stream.IOStream) (http.Handler, error) {
func (info *Info) Handler(route string, context context.Context, io stream.IOStream) (http.Handler, error) {
mux := http.NewServeMux()
// handle everything under /dis/!

3
internal/component/control/extras_self.go
View file

@ -1,6 +1,7 @@
package control
import (
"context"
"encoding/json"
"fmt"
"net/http"
@ -22,7 +23,7 @@ func (SelfHandler) Name() string { return "control-self" }
func (*SelfHandler) Routes() []string { return []string{"/"} }
func (sh *SelfHandler) Handler(route string, io stream.IOStream) (http.Handler, error) {
func (sh *SelfHandler) Handler(route string, context context.Context, io stream.IOStream) (http.Handler, error) {
// create a redirect
var redirect Redirect
var err error

9
internal/component/control/server.go
View file

@ -1,14 +1,17 @@
package control
import (
"context"
"net/http"
"github.com/tkw1536/goprogram/stream"
)
// Server returns an http.Mux that implements the main server instance
// Server returns an http.Mux that implements the main server instance.
// The server may spawn background tasks, but these should be terminated once context closes.
//
// Logging messages are directed to io.
func (control *Control) Server(io stream.IOStream) (*http.ServeMux, error) {
func (control *Control) Server(context context.Context, io stream.IOStream) (*http.ServeMux, error) {
// create a new mux
mux := http.NewServeMux()
@ -16,7 +19,7 @@ func (control *Control) Server(io stream.IOStream) (*http.ServeMux, error) {
for _, s := range control.Servables {
for _, route := range s.Routes() {
io.Printf("mounting %s\n", route)
handler, err := s.Handler(route, io)
handler, err := s.Handler(route, context, io)
if err != nil {
return nil, err
}

6
internal/component/instances/wisski_log.go
View file

@ -3,7 +3,7 @@ package instances
import (
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
"github.com/tkw1536/goprogram/lib/collection"
)
// SnapshotLogFor retrieves (and prunes) the SnapshotLog for the provided slug.
@ -14,7 +14,7 @@ func (instances *Instances) SnapshotLogFor(slug string) (snapshots []models.Snap
return nil, err
}
return slicesx.Filter(snapshots, func(s models.Snapshot) bool {
return collection.Filter(snapshots, func(s models.Snapshot) bool {
return s.Slug == slug
}), nil
}
@ -35,7 +35,7 @@ func (instances *Instances) SnapshotLog() ([]models.Snapshot, error) {
}
// partition out the snapshots that have been deleted!
parts := slicesx.Partition(snapshots, func(s models.Snapshot) bool {
parts := collection.Partition(snapshots, func(s models.Snapshot) bool {
_, err := instances.Core.Environment.Stat(s.Path)
return !environment.IsNotExist(err)
})

6
internal/component/instances/wisski_pathbuilders.go
View file

@ -12,7 +12,7 @@ var exportPathbuilderPHP string
// Pathbuilders returns the ids of all pathbuilders in consistent order.
func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
err = wisski.ExecPHPScript(stream.FromDebug(), &ids, exportPathbuilderPHP, "all_list")
err = wisski.ExecPHPScript(stream.FromNil(), &ids, exportPathbuilderPHP, "all_list")
slices.Sort(ids)
return
}
@ -20,12 +20,12 @@ func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
// Pathbuilder returns a single pathbuilder as xml.
// If it does not exist, it returns the empty string and nil error.
func (wisski *WissKI) Pathbuilder(id string) (xml string, err error) {
err = wisski.ExecPHPScript(stream.FromDebug(), &xml, exportPathbuilderPHP, "one_xml", id)
err = wisski.ExecPHPScript(stream.FromNil(), &xml, exportPathbuilderPHP, "one_xml", id)
return
}
// AllPathbuilders returns all pathbuilders serialized as xml
func (wisski *WissKI) AllPathbuilders() (pathbuilders map[string]string, err error) {
err = wisski.ExecPHPScript(stream.FromDebug(), &pathbuilders, exportPathbuilderPHP, "all_xml")
err = wisski.ExecPHPScript(stream.FromNil(), &pathbuilders, exportPathbuilderPHP, "all_xml")
return
}

4
internal/component/instances/wisski_php.go
View file

@ -132,10 +132,10 @@ func marshalPHP(data any) (string, error) {
var settingsPHP string
func (wisski *WissKI) GetSettingsPHP(key string) (value any, err error) {
err = wisski.ExecPHPScript(stream.FromDebug(), &value, settingsPHP, "get_setting", key)
err = wisski.ExecPHPScript(stream.FromNil(), &value, settingsPHP, "get_setting", key)
return
}
func (wisski *WissKI) SetSettingsPHP(key string, value any) error {
return wisski.ExecPHPScript(stream.FromDebug(), nil, settingsPHP, "set_setting", key, value)
return wisski.ExecPHPScript(stream.FromNil(), nil, settingsPHP, "set_setting", key, value)
}

12
internal/component/instances/wisski_prefix.go
View file

@ -6,7 +6,7 @@ import (
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/stream"
_ "embed"
@ -38,13 +38,13 @@ func (wisski *WissKI) Prefixes() ([]string, error) {
func (wisski *WissKI) dbPrefixes() (prefixes []string, err error) {
// get all the ugly prefixes
err = wisski.ExecPHPScript(stream.FromDebug(), &prefixes, listURIPrefixesPHP, "list_prefixes")
err = wisski.ExecPHPScript(stream.FromNil(), &prefixes, listURIPrefixesPHP, "list_prefixes")
if err != nil {
return nil, err
}
// filter out sequential prefixes
prefixes = slicesx.NonSequential(prefixes, func(prev, now string) bool {
prefixes = collection.NonSequential(prefixes, func(prev, now string) bool {
return strings.HasPrefix(now, prev)
})
@ -55,7 +55,7 @@ func (wisski *WissKI) dbPrefixes() (prefixes []string, err error) {
}
// filter out blocked prefixes
return slicesx.Filter(prefixes, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil
return collection.Filter(prefixes, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil
}
func (instances *Instances) blockedPrefixes() ([]string, error) {
@ -87,7 +87,7 @@ func (instances *Instances) blockedPrefixes() ([]string, error) {
}
func hasAnyPrefix(candidate string, prefixes []string) bool {
return slicesx.Any(
return collection.Any(
prefixes,
func(prefix string) bool {
return strings.HasPrefix(candidate, prefix)
@ -144,5 +144,5 @@ func (wisski *WissKI) UpdatePrefixes() error {
return err
}
return wisski.Metadata().SetAll(PrefixConfigKey, slicesx.AsAny(prefixes)...)
return wisski.Metadata().SetAll(PrefixConfigKey, collection.AsAny(prefixes)...)
}

4
internal/component/pool_meta.go
View file

@ -4,7 +4,7 @@ import (
"reflect"
"sync"
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/lib/reflectx"
)
@ -88,7 +88,7 @@ func (m meta) InitComponent(instance reflect.Value, all []Component) {
// assign the component fields
for field, eType := range m.CFields {
c := slicesx.First(all, func(c Component) bool {
c := collection.First(all, func(c Component) bool {
return reflect.TypeOf(c).AssignableTo(eType)
})

49
internal/component/resolver/resolver.go
View file

@ -1,6 +1,7 @@
package resolver
import (
"context"
"fmt"
"net/http"
"regexp"
@ -19,15 +20,17 @@ type Resolver struct {
Instances *instances.Instances
prefixes lazy.Lazy[map[string]string] // cached prefixes (from the server)
handler lazy.Lazy[wdresolve.ResolveHandler] // handler
prefixes lazy.Lazy[map[string]string] // cached prefixes (from the server)
RefreshInterval time.Duration
handler lazy.Lazy[wdresolve.ResolveHandler] // handler
}
func (*Resolver) Name() string { return "resolver" }
func (resolver *Resolver) Routes() []string { return []string{"/go/", "/wisski/get/"} }
func (resolver *Resolver) Handler(route string, io stream.IOStream) (http.Handler, error) {
func (resolver *Resolver) Handler(route string, context context.Context, io stream.IOStream) (http.Handler, error) {
var err error
return resolver.handler.Get(func() (p wdresolve.ResolveHandler) {
p.TrustXForwardedProto = true
@ -49,6 +52,8 @@ func (resolver *Resolver) Handler(route string, io stream.IOStream) (http.Handle
io.Printf("registering legacy domain %s\n", domain)
}
go resolver.updatePrefixes(io, context)
// resolve the prefixes
p.Resolver = resolvers.InOrder{
resolver,
@ -58,26 +63,44 @@ func (resolver *Resolver) Handler(route string, io stream.IOStream) (http.Handle
}), err
}
func (resolver *Resolver) updatePrefixes(io stream.IOStream, ctx context.Context) {
t := time.NewTicker(resolver.RefreshInterval)
defer t.Stop()
for {
select {
case <-t.C:
io.Println("resolver: Reloading prefixes from database")
prefixes, _ := resolver.AllPrefixes()
resolver.prefixes.Set(prefixes)
case <-ctx.Done():
return
}
}
}
func (resolver *Resolver) Target(uri string) string {
return wdresolve.PrefixTarget(resolver, uri)
}
// allow reloading prefixes from the server every minute
const prefixesRefresh = time.Minute
// Prefixes returns a cached list of prefixes
func (resolver *Resolver) Prefixes() (prefixes map[string]string) {
// reset the prefixes after a specific time, but only if requested
resolver.prefixes.ResetAfter(prefixesRefresh)
return resolver.prefixes.Get(resolver.freshPrefixes)
return resolver.prefixes.Get(func() map[string]string {
prefixes, _ := resolver.AllPrefixes()
return prefixes
})
}
func (resolver *Resolver) freshPrefixes() map[string]string {
// AllPrefixes returns a list of all prefixes from the server.
// Prefixes may be cached on the server
func (resolver *Resolver) AllPrefixes() (map[string]string, error) {
instances, err := resolver.Instances.All()
if err != nil {
return nil
return nil, err
}
gPrefixes := make(map[string]string)
var lastErr error
for _, instance := range instances {
if instance.NoPrefix() {
continue
@ -88,6 +111,7 @@ func (resolver *Resolver) freshPrefixes() map[string]string {
// => skip it!
prefixes, err := instance.PrefixesCached()
if err != nil {
lastErr = err
continue
}
@ -95,5 +119,6 @@ func (resolver *Resolver) freshPrefixes() map[string]string {
gPrefixes[p] = url
}
}
return gPrefixes
return gPrefixes, lastErr
}

3
internal/component/server.go
View file

@ -1,6 +1,7 @@
package component
import (
"context"
"net/http"
"github.com/tkw1536/goprogram/stream"
@ -14,5 +15,5 @@ type Servable interface {
Routes() []string
// Handler returns the handler for the requested route
Handler(route string, io stream.IOStream) (http.Handler, error)
Handler(route string, context context.Context, io stream.IOStream) (http.Handler, error)
}

4
internal/component/snapshots/snapshot.go
View file

@ -10,7 +10,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/component/instances"
"github.com/FAU-CDI/wisski-distillery/internal/models"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
@ -91,7 +91,7 @@ func (snapshot *Snapshot) makeParts(ios stream.IOStream, snapshots *Manager, ins
defer st.Stop()
// get all the components
comps := slicesx.FilterClone(snapshots.Snapshotable, func(sc component.Snapshotable) bool {
comps := collection.FilterClone(snapshots.Snapshotable, func(sc component.Snapshotable) bool {
return sc.SnapshotNeedsRunning() == needsRunning
})

2
internal/component/sql/connect.go
View file

@ -91,7 +91,7 @@ func (sql *SQL) QueryTable(silent bool, table string) (*gorm.DB, error) {
// WaitQueryTable waits for a connection to succeed via QueryTable
func (sql *SQL) WaitQueryTable() error {
// TODO: Establish a convention on when to wait for this!
n := stream.FromDebug()
n := stream.FromNil()
return wait.Wait(func() bool {
_, err := sql.QueryTable(true, models.InstanceTable)
n.EPrintf("[SQL.WaitQueryTable]: %s\n", err)

4
internal/component/sql/snapshot.go
View file

@ -8,9 +8,9 @@ import (
"github.com/tkw1536/goprogram/stream"
)
func (SQL) SnapshotNeedsRunning() bool { return false }
func (*SQL) SnapshotNeedsRunning() bool { return false }
func (SQL) SnapshotName() string { return "sql" }
func (*SQL) SnapshotName() string { return "sql" }
func (sql *SQL) Snapshot(wisski models.Instance, context component.StagingContext) error {
return context.AddDirectory(".", func() error {

6
internal/component/sql/sql.go
View file

@ -22,15 +22,15 @@ type SQL struct {
lazyNetwork lazy.Lazy[string]
}
func (SQL) Name() string {
func (*SQL) Name() string {
return "sql"
}
func (sql SQL) Path() string {
func (sql *SQL) Path() string {
return filepath.Join(sql.Core.Config.DeployRoot, "core", sql.Name())
}
func (SQL) Context(parent component.InstallationContext) component.InstallationContext {
func (*SQL) Context(parent component.InstallationContext) component.InstallationContext {
return parent
}

2
internal/component/sql/update.go
View file

@ -21,7 +21,7 @@ func (sql *SQL) Shell(io stream.IOStream, argv ...string) (int, error) {
// unsafeWaitShell waits for a connection via the database shell to succeed
func (sql *SQL) unsafeWaitShell() error {
n := stream.FromDebug()
n := stream.FromNil()
return wait.Wait(func() bool {
code, err := sql.Shell(n, "-e", "select 1;")
n.EPrintf("[SQL.unsafeWaitShell]: %d %s\n", code, err)

2
internal/component/triplestore/database.go
View file

@ -86,7 +86,7 @@ func (ts Triplestore) OpenRaw(method, url string, body interface{}, bodyName str
// Wait waits for the connection to the Triplestore to succeed.
// This is achieved using a polling strategy.
func (ts Triplestore) Wait() error {
n := stream.FromDebug()
n := stream.FromNil()
return wait.Wait(func() bool {
res, err := ts.OpenRaw("GET", "/rest/repositories", nil, "", "")
n.EPrintf("[Triplestore.Wait]: %s\n", err)

20
internal/core/params.go
View file

@ -1,6 +1,9 @@
package core
import (
"context"
"os"
"os/signal"
"path/filepath"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
@ -8,7 +11,8 @@ import (
// Params are used to initialize the excutable.
type Params struct {
ConfigPath string // ConfigPath is the path to the configuration file for the distillery
ConfigPath string // ConfigPath is the path to the configuration file for the distillery
Context context.Context // Context for the distillery
}
// ParamsFromEnv creates a new set of parameters from the environment.
@ -28,5 +32,19 @@ func ParamsFromEnv() (params Params, err error) {
// and add the configuration file name to it!
params.ConfigPath = filepath.Join(params.ConfigPath, ConfigFile)
// generate a new context
ctx, cancel := context.WithCancel(context.Background())
params.Context = ctx
// cancel the context on an interrupt
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
<-c
cancel()
}()
// and return the params!
return params, nil
}

4
internal/dis/component.go
View file

@ -44,7 +44,9 @@ func (dis *Distillery) register(context *component.PoolContext) []component.Comp
// Control server
ra[*control.Control](dis, context),
ra[*control.SelfHandler](dis, context),
ra[*resolver.Resolver](dis, context),
r(dis, context, func(resolver *resolver.Resolver) {
resolver.RefreshInterval = time.Minute
}),
ra[*control.Info](dis, context),
}
}

5
internal/dis/distillery.go
View file

@ -21,6 +21,9 @@ type Distillery struct {
// core holds the core of the distillery
component.Core
// internal context for the distillery
context context.Context
// Upstream holds information to connect to the various running
// distillery components.
//
@ -40,7 +43,7 @@ type Upstream struct {
// Context returns a new Context belonging to this distillery
func (dis *Distillery) Context() context.Context {
return context.Background()
return dis.context
}
//

1
internal/dis/init.go
View file

@ -21,6 +21,7 @@ var errOpenConfig = exit.Error{
// NewDistillery creates a new distillery from the provided flags
func NewDistillery(params core.Params, flags core.Flags, req core.Requirements) (dis *Distillery, err error) {
dis = &Distillery{
context: params.Context,
Core: component.Core{
Environment: environment.Native{},
},