rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wdcli: Use progress writer instead of IOStream

Browse source
This commit is contained in:
Tom Wiesing 2022-11-30 11:39:29 +01:00
parent 890022ae64
commit 3b78b06fff
No known key found for this signature in database
49 changed files with 396 additions and 393 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cmd/backup.go
View file

@ -41,12 +41,12 @@ func (bk backup) Run(context wisski_distillery.Context) error {
// prune old backups
if !bk.NoPrune {
defer logging.LogOperation(func() error {
return dis.Exporter().PruneExports(context.Context, context.IOStream)
}, context.IOStream, "Pruning old backups")
return dis.Exporter().PruneExports(context.Context, context.Stderr)
}, context.Stderr, "Pruning old backups")
}
// do the handling
err := dis.Exporter().MakeExport(context.Context, context.IOStream, exporter.ExportTask{
err := dis.Exporter().MakeExport(context.Context, context.Stderr, exporter.ExportTask{
Dest: bk.Positionals.Dest,
StagingOnly: bk.StagingOnly,

6
cmd/blind_update.go
View file

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
@ -9,7 +10,6 @@ import (
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// BlindUpdate is the 'blind_update' command
@ -51,8 +51,8 @@ func (bu blindUpdate) Run(context wisski_distillery.Context) error {
}
// and do the actual blind_update!
return status.StreamGroup(context.IOStream, bu.Parallel, func(instance *wisski.WissKI, str stream.IOStream) error {
return instance.Drush().Update(context.Context, str)
return status.WriterGroup(context.Stderr, bu.Parallel, func(instance *wisski.WissKI, writer io.Writer) error {
return instance.Drush().Update(context.Context, writer)
}, wissKIs, status.SmartMessage(func(item *wisski.WissKI) string {
return fmt.Sprintf("blind_update %q", item.Slug)
}))

12
cmd/bootstrap.go
View file

@ -84,7 +84,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
{
logging.LogMessage(context.IOStream, "Creating root deployment directory")
logging.LogMessage(context.Stderr, "Creating root deployment directory")
if err := env.MkdirAll(root, environment.DefaultDirPerm); err != nil {
return errBootstrapFailedToCreateDirectory.WithMessageF(root)
}
@ -109,7 +109,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
{
logging.LogMessage(context.IOStream, "Copying over wdcli executable")
logging.LogMessage(context.Stderr, "Copying over wdcli executable")
exe, err := env.Executable()
if err != nil {
return errBoostrapFailedToCopyExe.WithMessageF(err)
@ -132,7 +132,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
defer env.Close()
return tpl.MarshalTo(env)
}, context.IOStream, "Installing configuration file"); err != nil {
}, context.Stderr, "Installing configuration file"); err != nil {
return errBootstrapWriteConfig.WithMessageF(err)
}
@ -169,7 +169,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
return nil
}, context.IOStream, "Creating additional config files"); err != nil {
}, context.Stderr, "Creating additional config files"); err != nil {
return errBootstrapCreateFile.WithMessageF(err)
}
}
@ -177,7 +177,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
}
// re-read the configuration and print it!
logging.LogMessage(context.IOStream, "Configuration is now complete")
logging.LogMessage(context.Stderr, "Configuration is now complete")
f, err := env.Open(envPath)
if err != nil {
return errBootstrapOpenConfig.WithMessageF(err)
@ -191,7 +191,7 @@ func (bs cBootstrap) Run(context wisski_distillery.Context) error {
context.Println(cfg)
// Tell the user how to proceed
logging.LogMessage(context.IOStream, "Bootstrap is complete")
logging.LogMessage(context.Stderr, "Bootstrap is complete")
context.Printf("Adjust the configuration file at %s\n", envPath)
context.Printf("Then make sure 'docker compose' is installed.\n")
context.Printf("Finally grab a GraphDB zipped source file and run:\n")

6
cmd/cron.go
View file

@ -2,12 +2,12 @@ package cmd
import (
"fmt"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// Cron is the 'cron' command
@ -39,8 +39,8 @@ func (cr cron) Run(context wisski_distillery.Context) error {
}
// and do the actual blind_update!
return status.StreamGroup(context.IOStream, cr.Parallel, func(instance *wisski.WissKI, io stream.IOStream) error {
return instance.Drush().Cron(context.Context, io)
return status.WriterGroup(context.Stderr, cr.Parallel, func(instance *wisski.WissKI, writer io.Writer) error {
return instance.Drush().Cron(context.Context, writer)
}, wissKIs, status.SmartMessage(func(item *wisski.WissKI) string {
return fmt.Sprintf("cron %q", item.Slug)
}))

12
cmd/monday.go
View file

@ -39,36 +39,36 @@ func (monday monday) AfterParse() error {
func (monday monday) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
return context.Exec("backup")
}, context.IOStream, "Running backup"); err != nil {
}, context.Stderr, "Running backup"); err != nil {
return err
}
if err := logging.LogOperation(func() error {
return context.Exec("system_update", monday.Positionals.GraphdbZip)
}, context.IOStream, "Running system_update"); err != nil {
}, context.Stderr, "Running system_update"); err != nil {
return err
}
if err := logging.LogOperation(func() error {
return context.Exec("rebuild")
}, context.IOStream, "Running rebuild"); err != nil {
}, context.Stderr, "Running rebuild"); err != nil {
return err
}
if err := logging.LogOperation(func() error {
return context.Exec("update_prefix_config")
}, context.IOStream, "Running update_prefix_config"); err != nil {
}, context.Stderr, "Running update_prefix_config"); err != nil {
return err
}
if monday.UpdateInstances {
if err := logging.LogOperation(func() error {
return context.Exec("blind_update")
}, context.IOStream, "Running blind_update"); err != nil {
}, context.Stderr, "Running blind_update"); err != nil {
return err
}
}
logging.LogMessage(context.IOStream, "Done, have a great week!")
logging.LogMessage(context.Stderr, "Done, have a great week!")
return nil
}

20
cmd/provision.go
View file

@ -44,7 +44,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
slug := p.Positionals.Slug
// check that it doesn't already exist
logging.LogMessage(context.IOStream, "Provisioning new WissKI instance %s", slug)
logging.LogMessage(context.Stderr, "Provisioning new WissKI instance %s", slug)
if exists, err := dis.Instances().Has(context.Context, slug); err != nil || exists {
return errProvisionAlreadyExists.WithMessageF(slug)
}
@ -56,7 +56,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
}
// check that the base directory does not exist
logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
logging.LogMessage(context.Stderr, "Checking that base directory %s does not exist", instance.FilesystemBase)
if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) {
return errProvisionAlreadyExists.WithMessageF(slug)
}
@ -68,7 +68,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
}
return nil
}, context.IOStream, "Updating bookkeeping database"); err != nil {
}, context.Stderr, "Updating bookkeeping database"); err != nil {
return err
}
@ -76,7 +76,7 @@ func (p provision) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
domain := instance.Domain()
for _, pc := range dis.Provisionable() {
logging.LogMessage(context.IOStream, "Provisioning %s resources", pc.Name())
logging.LogMessage(context.Stderr, "Provisioning %s resources", pc.Name())
err := pc.Provision(context.Context, instance.Instance, domain)
if err != nil {
return err
@ -84,29 +84,29 @@ func (p provision) Run(context wisski_distillery.Context) error {
}
return nil
}, context.IOStream, "Provisioning instance-specific resources"); err != nil {
}, context.Stderr, "Provisioning instance-specific resources"); err != nil {
return errProvisionGeneric.WithMessageF(slug, err)
}
// run the provision script
if err := logging.LogOperation(func() error {
if err := instance.Provisioner().Provision(context.Context, context.IOStream); err != nil {
if err := instance.Provisioner().Provision(context.Context, context.Stderr); err != nil {
return errProvisionGeneric.WithMessageF(slug, err)
}
return nil
}, context.IOStream, "Running setup scripts"); err != nil {
}, context.Stderr, "Running setup scripts"); err != nil {
return err
}
// start the container!
logging.LogMessage(context.IOStream, "Starting Container")
if err := instance.Barrel().Stack().Up(context.Context, context.IOStream); err != nil {
logging.LogMessage(context.Stderr, "Starting Container")
if err := instance.Barrel().Stack().Up(context.Context, context.Stderr); err != nil {
return err
}
// and we're done!
logging.LogMessage(context.IOStream, "Instance has been provisioned")
logging.LogMessage(context.Stderr, "Instance has been provisioned")
context.Printf("URL: %s\n", instance.URL().String())
context.Printf("Username: %s\n", instance.DrupalUsername)
context.Printf("Password: %s\n", instance.DrupalPassword)

18
cmd/purge.go
View file

@ -58,7 +58,7 @@ func (p purge) Run(context wisski_distillery.Context) error {
}
// load the instance (first via bookkeeping, then via defaults)
logging.LogMessage(context.IOStream, "Checking bookkeeping table")
logging.LogMessage(context.Stderr, "Checking bookkeeping table")
instance, err := dis.Instances().WissKI(context.Context, slug)
if err == instances.ErrWissKINotFound {
context.Println("Not found in bookkeeping table, assuming defaults")
@ -69,13 +69,13 @@ func (p purge) Run(context wisski_distillery.Context) error {
}
// remove docker stack
logging.LogMessage(context.IOStream, "Stopping and removing docker container")
if err := instance.Barrel().Stack().Down(context.Context, context.IOStream); err != nil {
logging.LogMessage(context.Stderr, "Stopping and removing docker container")
if err := instance.Barrel().Stack().Down(context.Context, context.Stderr); err != nil {
context.EPrintln(err)
}
// remove the filesystem
logging.LogMessage(context.IOStream, "Removing from filesystem %s", instance.FilesystemBase)
logging.LogMessage(context.Stderr, "Removing from filesystem %s", instance.FilesystemBase)
if err := dis.Still.Environment.RemoveAll(instance.FilesystemBase); err != nil {
context.EPrintln(err)
}
@ -84,7 +84,7 @@ func (p purge) Run(context wisski_distillery.Context) error {
if err := logging.LogOperation(func() error {
domain := instance.Domain()
for _, pc := range dis.Provisionable() {
logging.LogMessage(context.IOStream, "Purging %s resources", pc.Name())
logging.LogMessage(context.Stderr, "Purging %s resources", pc.Name())
err := pc.Purge(context.Context, instance.Instance, domain)
if err != nil {
return err
@ -92,22 +92,22 @@ func (p purge) Run(context wisski_distillery.Context) error {
}
return nil
}, context.IOStream, "Purging instance-specific resources"); err != nil {
}, context.Stderr, "Purging instance-specific resources"); err != nil {
return errPurgeGeneric.WithMessageF(slug, err)
}
// remove from bookkeeping
logging.LogMessage(context.IOStream, "Removing instance from bookkeeping")
logging.LogMessage(context.Stderr, "Removing instance from bookkeeping")
if err := instance.Bookkeeping().Delete(context.Context); err != nil {
context.EPrintln(err)
}
// remove the filesystem
logging.LogMessage(context.IOStream, "Remove lock data")
logging.LogMessage(context.Stderr, "Remove lock data")
if instance.Locker().TryUnlock(context.Context) {
context.EPrintln("instance was not locked")
}
logging.LogMessage(context.IOStream, "Instance %s has been purged", slug)
logging.LogMessage(context.Stderr, "Instance %s has been purged", slug)
return nil
}

6
cmd/rebuild.go
View file

@ -2,13 +2,13 @@ package cmd
import (
"fmt"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// Cron is the 'cron' command
@ -46,8 +46,8 @@ func (rb rebuild) Run(context wisski_distillery.Context) error {
}
// and do the actual rebuild
return status.StreamGroup(context.IOStream, rb.Parallel, func(instance *wisski.WissKI, io stream.IOStream) error {
return instance.Barrel().Build(context.Context, io, true)
return status.WriterGroup(context.Stderr, rb.Parallel, func(instance *wisski.WissKI, writer io.Writer) error {
return instance.Barrel().Build(context.Context, writer, true)
}, wissKIs, status.SmartMessage(func(item *wisski.WissKI) string {
return fmt.Sprintf("rebuild %q", item.Slug)
}))

14
cmd/reserve.go
View file

@ -45,7 +45,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
slug := r.Positionals.Slug
// check that it doesn't already exist
logging.LogMessage(context.IOStream, "Reserving new WissKI instance %s", slug)
logging.LogMessage(context.Stderr, "Reserving new WissKI instance %s", slug)
if exists, err := dis.Instances().Has(context.Context, slug); err != nil || exists {
return errProvisionAlreadyExists.WithMessageF(slug)
}
@ -57,7 +57,7 @@ func (r reserve) Run(context wisski_distillery.Context) error {
}
// check that the base directory does not exist
logging.LogMessage(context.IOStream, "Checking that base directory %s does not exist", instance.FilesystemBase)
logging.LogMessage(context.Stderr, "Checking that base directory %s does not exist", instance.FilesystemBase)
if fsx.IsDirectory(dis.Environment, instance.FilesystemBase) {
return errProvisionAlreadyExists.WithMessageF(slug)
}
@ -66,20 +66,20 @@ func (r reserve) Run(context wisski_distillery.Context) error {
s := instance.Reserve().Stack()
{
if err := logging.LogOperation(func() error {
return s.Install(context.Context, context.IOStream, component.InstallationContext{})
}, context.IOStream, "Installing docker stack"); err != nil {
return s.Install(context.Context, context.Stderr, component.InstallationContext{})
}, context.Stderr, "Installing docker stack"); err != nil {
return err
}
if err := logging.LogOperation(func() error {
return s.Update(context.Context, context.IOStream, true)
}, context.IOStream, "Updating docker stack"); err != nil {
return s.Update(context.Context, context.Stderr, true)
}, context.Stderr, "Updating docker stack"); err != nil {
return err
}
}
// and we're done!
logging.LogMessage(context.IOStream, "Instance has been reserved")
logging.LogMessage(context.Stderr, "Instance has been reserved")
context.Printf("URL: %s\n", instance.URL().String())
return nil

2
cmd/server.go
View file

@ -34,7 +34,7 @@ var errServerListen = exit.Error{
func (s server) Run(context wisski_distillery.Context) error {
dis := context.Environment
handler, err := dis.Control().Server(context.Context, context.IOStream)
handler, err := dis.Control().Server(context.Context, context.Stderr)
if err != nil {
return err
}

2
cmd/snapshot.go
View file

@ -45,7 +45,7 @@ func (sn snapshot) Run(context wisski_distillery.Context) error {
}
// do a snapshot of it!
err = dis.Exporter().MakeExport(context.Context, context.IOStream, exporter.ExportTask{
err = dis.Exporter().MakeExport(context.Context, context.Stderr, exporter.ExportTask{
Dest: sn.Positionals.Dest,
StagingOnly: sn.StagingOnly,

2
cmd/ssh.go
View file

@ -31,7 +31,7 @@ var errSSHListen = exit.Error{
func (s ssh) Run(context wisski_distillery.Context) error {
dis := context.Environment
server, err := dis.SSH().Server(context.Context, s.PrivateKeyPath, context.IOStream)
server, err := dis.SSH().Server(context.Context, s.PrivateKeyPath, context.Stderr)
if err != nil {
return err
}

74
cmd/system_pause.go
View file

@ -12,7 +12,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// SystemPause is the 'system_pause' command
@ -54,24 +53,23 @@ func (sp systempause) Run(context wisski_distillery.Context) error {
}
func (sp systempause) start(context wisski_distillery.Context, dis *dis.Distillery) error {
logging.LogMessage(context.IOStream, "Starting Components")
logging.LogMessage(context.Stderr, "Starting Components")
// find all the core stacks
if err := status.RunErrorGroup(context.Stdout, status.Group[component.Installable, error]{
if err := status.RunErrorGroup(context.Stderr, status.Group[component.Installable, error]{
PrefixString: func(item component.Installable, index int) string {
return fmt.Sprintf("[up %q]: ", item.Name())
},
PrefixAlign: true,
Handler: func(item component.Installable, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
return item.Stack(context.Environment.Environment).Up(context.Context, io)
return item.Stack(context.Environment.Environment).Up(context.Context, writer)
},
}, dis.Installable()); err != nil {
return err
}
logging.LogMessage(context.IOStream, "Starting Up WissKIs")
logging.LogMessage(context.Stderr, "Starting Up WissKIs")
// find the instances
wissKIs, err := dis.Instances().All(context.Context)
@ -80,15 +78,14 @@ func (sp systempause) start(context wisski_distillery.Context, dis *dis.Distille
}
// shut them all down
if err := status.RunErrorGroup(context.Stdout, status.Group[*wisski.WissKI, error]{
if err := status.RunErrorGroup(context.Stderr, status.Group[*wisski.WissKI, error]{
PrefixString: func(item *wisski.WissKI, index int) string {
return fmt.Sprintf("[up %q]: ", item.Slug)
},
PrefixAlign: true,
Handler: func(item *wisski.WissKI, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
return item.Barrel().Stack().Up(context.Context, io)
return item.Barrel().Stack().Up(context.Context, writer)
},
}, wissKIs); err != nil {
return err
@ -98,7 +95,7 @@ func (sp systempause) start(context wisski_distillery.Context, dis *dis.Distille
}
func (sp systempause) stop(context wisski_distillery.Context, dis *dis.Distillery) error {
logging.LogMessage(context.IOStream, "Shutting Down WissKIs")
logging.LogMessage(context.Stderr, "Shutting Down WissKIs")
// find the instances
wissKIs, err := dis.Instances().All(context.Context)
@ -107,32 +104,30 @@ func (sp systempause) stop(context wisski_distillery.Context, dis *dis.Distiller
}
// shut them all down
if err := status.RunErrorGroup(context.Stdout, status.Group[*wisski.WissKI, error]{
if err := status.RunErrorGroup(context.Stderr, status.Group[*wisski.WissKI, error]{
PrefixString: func(item *wisski.WissKI, index int) string {
return fmt.Sprintf("[down %q]: ", item.Slug)
},
PrefixAlign: true,
Handler: func(item *wisski.WissKI, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
return item.Barrel().Stack().Down(context.Context, io)
return item.Barrel().Stack().Down(context.Context, writer)
},
}, wissKIs); err != nil {
return err
}
logging.LogMessage(context.IOStream, "Shutting Down Components")
logging.LogMessage(context.Stderr, "Shutting Down Components")
// find all the core stacks
if err := status.RunErrorGroup(context.Stdout, status.Group[component.Installable, error]{
if err := status.RunErrorGroup(context.Stderr, status.Group[component.Installable, error]{
PrefixString: func(item component.Installable, index int) string {
return fmt.Sprintf("[down %q]: ", item.Name())
},
PrefixAlign: true,
Handler: func(item component.Installable, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
return item.Stack(context.Environment.Environment).Down(context.Context, io)
return item.Stack(context.Environment.Environment).Down(context.Context, writer)
},
}, dis.Installable()); err != nil {
return err
@ -140,48 +135,3 @@ func (sp systempause) stop(context wisski_distillery.Context, dis *dis.Distiller
return nil
}
/*
handleStack := func(io stream.IOStream, stack component.StackWithResources) error {
if sp.Start {
return stack.Up(io)
} else {
return stack.Down(io)
}
}
logging.LogMessage(context.IOStream, "Iterating over components")
if err := status.RunErrorGroup(context.Stdout, status.Group[component.Installable, error]{
PrefixString: func(item component.Installable, index int) string {
return fmt.Sprintf("[%s %q]: ", verb, item.Name())
},
PrefixAlign: true,
Handler: func(item component.Installable, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
stack := item.Stack(context.Environment.Environment)
return handleStack(io, stack)
},
}, dis.Installable()); err != nil {
return err
}
logging.LogMessage(context.IOStream, "Shutting Down WissKIs")
// find the instances
wissKIs, err := dis.Instances().All()
if err != nil {
return err
}
// and do the actual rebuild
if err := status.StreamGroup(context.IOStream, rb.Parallel, func(instance *wisski.WissKI, io stream.IOStream) error {
return instance.Barrel().Build(io, true)
}, wissKIs, status.SmartMessage(func(item *wisski.WissKI) string {
return fmt.Sprintf("rebuild %q", item.Slug)
})); err != nil {
return err
}
}
*/

32
cmd/system_update.go
View file

@ -14,7 +14,6 @@ import (
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/parser"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// SystemUpdate is the 'system_update' command
@ -67,7 +66,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
dis := context.Environment
// create all the other directories
logging.LogMessage(context.IOStream, "Ensuring distillery installation directories exist")
logging.LogMessage(context.Stderr, "Ensuring distillery installation directories exist")
for _, d := range []string{
dis.Config.DeployRoot,
dis.Instances().Path(),
@ -82,7 +81,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
if si.InstallDocker {
// install system updates
logging.LogMessage(context.IOStream, "Updating Operating System Packages")
logging.LogMessage(context.Stderr, "Updating Operating System Packages")
if err := si.mustExec(context, "", "apt-get", "update"); err != nil {
return err
}
@ -91,7 +90,7 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
}
// install docker
logging.LogMessage(context.IOStream, "Installing / Updating Docker")
logging.LogMessage(context.Stderr, "Installing / Updating Docker")
if err := si.mustExec(context, "", "apt-get", "install", "curl"); err != nil {
return err
}
@ -101,19 +100,19 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
}
}
logging.LogMessage(context.IOStream, "Checking that 'docker' is installed")
logging.LogMessage(context.Stderr, "Checking that 'docker' is installed")
if err := si.mustExec(context, "", "docker", "--version", dis.Config.DockerNetworkName); err != nil {
return err
}
logging.LogMessage(context.IOStream, "Checking that 'docker compose' is available")
logging.LogMessage(context.Stderr, "Checking that 'docker compose' is available")
if err := si.mustExec(context, "", "docker", "compose", "version"); err != nil {
return err
}
// create the docker network
// TODO: Use docker API for this
logging.LogMessage(context.IOStream, "Updating Docker Configuration")
logging.LogMessage(context.Stderr, "Updating Docker Configuration")
si.mustExec(context, "", "docker", "network", "create", dis.Config.DockerNetworkName)
// install and update the various stacks!
@ -125,21 +124,20 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
var updateMutex sync.Mutex
if err := logging.LogOperation(func() error {
return status.RunErrorGroup(context.Stdout, status.Group[component.Installable, error]{
return status.RunErrorGroup(context.Stderr, status.Group[component.Installable, error]{
PrefixString: func(item component.Installable, index int) string {
return fmt.Sprintf("[update %q]: ", item.Name())
},
PrefixAlign: true,
Handler: func(item component.Installable, index int, writer io.Writer) error {
io := stream.NewIOStream(writer, writer, stream.Null, 0)
stack := item.Stack(context.Environment.Environment)
if err := stack.Install(context.Context, io, item.Context(ctx)); err != nil {
if err := stack.Install(context.Context, writer, item.Context(ctx)); err != nil {
return err
}
if err := stack.Update(context.Context, io, true); err != nil {
if err := stack.Update(context.Context, writer, true); err != nil {
return err
}
@ -154,10 +152,10 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
updated[item.ID()] = struct{}{}
}()
return ud.Update(context.Context, io)
return ud.Update(context.Context, writer)
},
}, dis.Installable())
}, context.IOStream, "Performing Stack Updates"); err != nil {
}, context.Stderr, "Performing Stack Updates"); err != nil {
return err
}
@ -170,18 +168,18 @@ func (si systemupdate) Run(context wisski_distillery.Context) error {
context.Println("Already updated")
return nil
}
return item.Update(context.Context, context.IOStream)
}, context.IOStream, "Updating Component: %s", name); err != nil {
return item.Update(context.Context, context.Stderr)
}, context.Stderr, "Updating Component: %s", name); err != nil {
return errBootstrapComponent.WithMessageF(name, err)
}
}
return nil
}, context.IOStream, "Performing Component Updates"); err != nil {
}, context.Stderr, "Performing Component Updates"); err != nil {
return err
}
// TODO: Register cronjob in /etc/cron.d!
logging.LogMessage(context.IOStream, "System has been updated")
logging.LogMessage(context.Stderr, "System has been updated")
return nil
}

6
cmd/update_prefix_config.go
View file

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"io"
wisski_distillery "github.com/FAU-CDI/wisski-distillery"
"github.com/FAU-CDI/wisski-distillery/internal/cli"
@ -9,7 +10,6 @@ import (
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// Cron is the 'cron' command
@ -42,8 +42,8 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error {
return errPrefixUpdateFailed.Wrap(err)
}
return status.StreamGroup(context.IOStream, upc.Parallel, func(instance *wisski.WissKI, io stream.IOStream) error {
io.Println("reading prefixes")
return status.WriterGroup(context.Stderr, upc.Parallel, func(instance *wisski.WissKI, writer io.Writer) error {
fmt.Fprintln(writer, "reading prefixes")
err := instance.Prefixes().Update(context.Context)
if err != nil {
return errPrefixUpdateFailed.Wrap(err)

4
go.mod
View file

@ -12,10 +12,11 @@ require (
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.5.0
github.com/pkg/errors v0.9.1
github.com/tkw1536/goprogram v0.2.0
github.com/tkw1536/goprogram v0.2.1
golang.org/x/crypto v0.3.0
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9
golang.org/x/sync v0.1.0
golang.org/x/term v0.2.0
gorm.io/driver/mysql v1.4.4
gorm.io/gorm v1.24.2
)
@ -28,5 +29,4 @@ require (
github.com/jinzhu/now v1.1.5 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
golang.org/x/sys v0.2.0 // indirect
golang.org/x/term v0.2.0 // indirect
)

14
go.sum
View file

@ -29,21 +29,15 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/tkw1536/goprogram v0.1.1 h1:gamK9OuRqoX2yQlA/nkgfVHHZWd/u2uUj6vJMYrYa70=
github.com/tkw1536/goprogram v0.1.1/go.mod h1:Jqs0sTMzhrAGCX3JQrlEwQ0WRWQACCvuQQkaBDp65pE=
github.com/tkw1536/goprogram v0.2.0 h1:qoa5Izgq5gfVggkAcOtCwpjz4oZv1KgDEJ8BHIK/djQ=
github.com/tkw1536/goprogram v0.2.0/go.mod h1:Jqs0sTMzhrAGCX3JQrlEwQ0WRWQACCvuQQkaBDp65pE=
github.com/tkw1536/goprogram v0.2.1 h1:7HtFn52iqVWtv7VQUgyU3apH30b0k3b4jAtwz0O9KOU=
github.com/tkw1536/goprogram v0.2.1/go.mod h1:Jqs0sTMzhrAGCX3JQrlEwQ0WRWQACCvuQQkaBDp65pE=
golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741 h1:fGZugkZk2UgYBxtpKmvub51Yno1LJDeEsRp2xGD+0gY=
golang.org/x/exp v0.0.0-20221004215720-b9f4876ce741/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9 h1:yZNXmy+j/JpX19vZkVktWqAo7Gny4PBWYYK3zskGpx4=
golang.org/x/exp v0.0.0-20221126150942-6ab00d035af9/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY=
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -63,12 +57,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM=
gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
gorm.io/driver/mysql v1.4.4 h1:MX0K9Qvy0Na4o7qSC/YI7XxqUw5KDw01umqgID+svdQ=
gorm.io/driver/mysql v1.4.4/go.mod h1:BCg8cKI+R0j/rZRQxeKis/forqRwRSYOR8OM3Wo6hOM=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.10 h1:4Ne9ZbzID9GUxRkllxN4WjJKpsHx8YbKvekVdgyWh24=
gorm.io/gorm v1.23.10/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/gorm v1.24.2 h1:9wR6CFD+G8nOusLdvkZelOEhpJVwwHzpQOUM+REd6U0=
gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=

18
internal/dis/component/backup.go
View file

@ -2,6 +2,7 @@ package component
import (
"context"
"fmt"
"io"
"path/filepath"
@ -9,7 +10,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
// Backupable represents a component with a Backup method
@ -39,8 +39,8 @@ type Snapshotable interface {
// StagingContext represents a context for [Backupable] and [Snapshotable]
type StagingContext interface {
// IO returns the input output stream belonging to this backup file
IO() stream.IOStream
// Progress returns a writer to write progress information to.
Progress() io.Writer
// Name creates a new directory inside the destination.
// Passing the empty path creates the destination as a directory.
@ -66,11 +66,11 @@ type StagingContext interface {
}
// NewStagingContext returns a new [StagingContext]
func NewStagingContext(ctx context.Context, env environment.Environment, io stream.IOStream, path string, manifest chan<- string) StagingContext {
func NewStagingContext(ctx context.Context, env environment.Environment, progress io.Writer, path string, manifest chan<- string) StagingContext {
return &stagingContext{
ctx: ctx,
env: env,
io: io,
progress: progress,
path: path,
manifest: manifest,
}
@ -80,7 +80,7 @@ func NewStagingContext(ctx context.Context, env environment.Environment, io stre
type stagingContext struct {
ctx context.Context
env environment.Environment // environment
io stream.IOStream // context the files are sent to
progress io.Writer // writer to direct progress to
path string // path to send files to
manifest chan<- string // channel the manifest is sent to
}
@ -93,12 +93,12 @@ func (bc *stagingContext) sendPath(path string) {
return
}
bc.io.Println(dst)
fmt.Fprintln(bc.progress, dst)
bc.manifest <- dst
}
func (bc *stagingContext) IO() stream.IOStream {
return bc.io
func (bc *stagingContext) Progress() io.Writer {
return bc.progress
}
var errResolveAbsolute = errors.New("resolve: path must be relative")

10
internal/dis/component/control/home/home.go
View file

@ -3,13 +3,13 @@ package home
import (
"context"
"fmt"
"io"
"net/http"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
"github.com/tkw1536/goprogram/stream"
)
type Home struct {
@ -30,10 +30,10 @@ var (
func (*Home) Routes() []string { return []string{"/"} }
func (home *Home) Handler(ctx context.Context, route string, io stream.IOStream) (http.Handler, error) {
home.updateRedirect(ctx, io)
home.updateInstances(ctx, io)
home.updateRender(ctx, io)
func (home *Home) Handler(ctx context.Context, route string, progress io.Writer) (http.Handler, error) {
home.updateRedirect(ctx, progress)
home.updateInstances(ctx, progress)
home.updateRender(ctx, progress)
return home, nil
}

15
internal/dis/component/control/home/public.go
View file

@ -3,6 +3,8 @@ package home
import (
"bytes"
"context"
"fmt"
"io"
"time"
_ "embed"
@ -10,14 +12,13 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/control/static"
"github.com/FAU-CDI/wisski-distillery/internal/status"
"github.com/FAU-CDI/wisski-distillery/pkg/timex"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/sync/errgroup"
)
func (home *Home) updateInstances(ctx context.Context, io stream.IOStream) {
func (home *Home) updateInstances(ctx context.Context, progress io.Writer) {
go func() {
for t := range timex.TickContext(ctx, home.RefreshInterval) {
io.Printf("[%s]: reloading instance list\n", t.Format(time.Stamp))
fmt.Fprintf(progress, "[%s]: reloading instance list\n", t.Format(time.Stamp))
err := (func() error {
ctx, cancel := context.WithTimeout(ctx, home.RefreshInterval)
@ -32,7 +33,7 @@ func (home *Home) updateInstances(ctx context.Context, io stream.IOStream) {
return nil
})()
if err != nil {
io.EPrintf("error reloading instances: ", err.Error())
fmt.Fprintf(progress, "error reloading instances: %s", err.Error())
}
}
}()
@ -51,10 +52,10 @@ func (home *Home) instanceMap(ctx context.Context) (map[string]struct{}, error)
return names, nil
}
func (home *Home) updateRender(ctx context.Context, io stream.IOStream) {
func (home *Home) updateRender(ctx context.Context, progress io.Writer) {
go func() {
for t := range timex.TickContext(ctx, home.RefreshInterval) {
io.Printf("[%s]: reloading home render list\n", t.Format(time.Stamp))
fmt.Fprintf(progress, "[%s]: reloading home render list\n", t.Format(time.Stamp))
err := (func() error {
ctx, cancel := context.WithTimeout(ctx, home.RefreshInterval)
@ -69,7 +70,7 @@ func (home *Home) updateRender(ctx context.Context, io stream.IOStream) {
return nil
})()
if err != nil {
io.EPrintf("error reloading instances: ", err.Error())
fmt.Fprintf(progress, "error reloading instances: %s", err.Error())
}
}
}()

9
internal/dis/component/control/home/redirect.go
View file

@ -3,18 +3,19 @@ package home
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/timex"
"github.com/tkw1536/goprogram/stream"
)
func (home *Home) updateRedirect(ctx context.Context, io stream.IOStream) {
func (home *Home) updateRedirect(ctx context.Context, progress io.Writer) {
go func() {
for t := range timex.TickContext(ctx, home.RefreshInterval) {
io.Printf("[%s]: reloading overrides\n", t.Format(time.Stamp))
fmt.Fprintf(progress, "[%s]: reloading overrides\n", t.Format(time.Stamp))
err := (func() error {
ctx, cancel := context.WithTimeout(ctx, home.RefreshInterval)
@ -29,7 +30,7 @@ func (home *Home) updateRedirect(ctx context.Context, io stream.IOStream) {
return nil
})()
if err != nil {
io.EPrintf("error reloading overrides: ", err.Error())
fmt.Fprintf(progress, "error reloading overrides: %s", err.Error())
}
}

4
internal/dis/component/control/info/info.go
View file

@ -2,6 +2,7 @@ package info
import (
"context"
"io"
"net/http"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
@ -12,7 +13,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
"github.com/tkw1536/goprogram/stream"
)
type Info struct {
@ -33,7 +33,7 @@ var (
func (*Info) Routes() []string { return []string{"/dis/"} }
func (info *Info) Handler(ctx context.Context, route string, io stream.IOStream) (handler http.Handler, err error) {
func (info *Info) Handler(ctx context.Context, route string, progress io.Writer) (handler http.Handler, err error) {
router := mux.NewRouter()
{
socket := &httpx.WebSocket{

36
internal/dis/component/control/info/socket.go
View file

@ -3,28 +3,29 @@ package info
import (
"context"
"encoding/json"
"fmt"
"io"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/exporter"
"github.com/FAU-CDI/wisski-distillery/internal/wisski"
"github.com/FAU-CDI/wisski-distillery/pkg/httpx"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
type InstanceAction struct {
NumParams int
HandleInteractive func(ctx context.Context, info *Info, instance *wisski.WissKI, str stream.IOStream, params ...string) error
HandleInteractive func(ctx context.Context, info *Info, instance *wisski.WissKI, out io.Writer, params ...string) error
HandleResult func(ctx context.Context, info *Info, instance *wisski.WissKI, params ...string) (value any, err error)
}
var socketInstanceActions = map[string]InstanceAction{
"snapshot": {
HandleInteractive: func(ctx context.Context, info *Info, instance *wisski.WissKI, str stream.IOStream, params ...string) error {
HandleInteractive: func(ctx context.Context, info *Info, instance *wisski.WissKI, out io.Writer, params ...string) error {
return info.Exporter.MakeExport(
ctx,
str,
out,
exporter.ExportTask{
Dest: "",
Instance: instance,
@ -35,17 +36,17 @@ var socketInstanceActions = map[string]InstanceAction{
},
},
"rebuild": {
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, str stream.IOStream, params ...string) error {
return instance.Barrel().Build(ctx, str, true)
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, out io.Writer, params ...string) error {
return instance.Barrel().Build(ctx, out, true)
},
},
"update": {
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, str stream.IOStream, params ...string) error {
return instance.Drush().Update(ctx, str)
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, out io.Writer, params ...string) error {
return instance.Drush().Update(ctx, out)
},
},
"cron": {
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, str stream.IOStream, params ...string) error {
HandleInteractive: func(ctx context.Context, _ *Info, instance *wisski.WissKI, str io.Writer, params ...string) error {
return instance.Drush().Cron(ctx, str)
},
},
@ -108,33 +109,30 @@ func (info *Info) handleInstanceAction(conn httpx.WebSocketConnection, action In
}
defer writer.Close()
str := stream.NewIOStream(writer, writer, nil, 0)
// handle the interactive action
if action.HandleInteractive != nil {
err := action.HandleInteractive(conn.Context(), info, instance, str, params...)
err := action.HandleInteractive(conn.Context(), info, instance, writer, params...)
if err != nil {
str.EPrintln(err)
fmt.Fprintln(writer, err)
return
}
str.Println("done")
fmt.Fprintln(writer, "done")
}
// handle the result computation
if action.HandleResult != nil {
result, err := action.HandleResult(conn.Context(), info, instance, params...)
if err != nil {
str.Println("false")
fmt.Fprintln(writer, "false")
return
}
data, err := json.Marshal(result)
if err != nil {
str.Println("false")
fmt.Fprintln(writer, "false")
return
}
data = append(data, "\n"...)
str.Println("true")
str.Stdout.Write(data)
fmt.Fprintln(writer, "true")
fmt.Fprintln(writer, data)
}
}

12
internal/dis/component/control/server.go
View file

@ -2,24 +2,24 @@ package control
import (
"context"
"fmt"
"io"
"net/http"
"github.com/tkw1536/goprogram/stream"
)
// Server returns an http.Mux that implements the main server instance.
// The server may spawn background tasks, but these should be terminated once context closes.
//
// Logging messages are directed to io.
func (control *Control) Server(ctx context.Context, io stream.IOStream) (*http.ServeMux, error) {
// Logging messages are directed to progress
func (control *Control) Server(ctx context.Context, progress io.Writer) (*http.ServeMux, error) {
// create a new mux
mux := http.NewServeMux()
// add all the servable routes!
for _, s := range control.Servables {
for _, route := range s.Routes() {
io.Printf("mounting %s\n", route)
handler, err := s.Handler(ctx, route, io)
fmt.Fprintf(progress, "mounting %s\n", route)
handler, err := s.Handler(ctx, route, progress)
if err != nil {
return nil, err
}

4
internal/dis/component/control/static/static.go
View file

@ -4,11 +4,11 @@ package static
import (
"context"
"embed"
"io"
"io/fs"
"net/http"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/tkw1536/goprogram/stream"
)
type Static struct {
@ -24,7 +24,7 @@ func (*Static) Routes() []string { return []string{"/static/"} }
//go:embed dist
var staticFS embed.FS
func (static *Static) Handler(ctx context.Context, route string, io stream.IOStream) (http.Handler, error) {
func (static *Static) Handler(ctx context.Context, route string, progress io.Writer) (http.Handler, error) {
// take the filesystem
fs, err := fs.Sub(staticFS, "dist")
if err != nil {

21
internal/dis/component/exporter/backup.go
View file

@ -13,7 +13,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
)
@ -50,7 +49,7 @@ type BackupDescription struct {
}
// New create a new Backup
func (exporter *Exporter) NewBackup(ctx context.Context, io stream.IOStream, description BackupDescription) (backup Backup) {
func (exporter *Exporter) NewBackup(ctx context.Context, progress io.Writer, description BackupDescription) (backup Backup) {
backup.Description = description
// catch anything critical that happened during the snapshot
@ -61,16 +60,16 @@ func (exporter *Exporter) NewBackup(ctx context.Context, io stream.IOStream, des
// do the create keeping track of time!
logging.LogOperation(func() error {
backup.StartTime = time.Now().UTC()
backup.run(ctx, io, exporter)
backup.run(ctx, progress, exporter)
backup.EndTime = time.Now().UTC()
return nil
}, io, "Writing backup files")
}, progress, "Writing backup files")
return
}
func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Exporter) {
func (backup *Backup) run(ctx context.Context, progress io.Writer, exporter *Exporter) {
// create a manifest
manifest, done := backup.handleManifest(backup.Description.Dest)
defer done()
@ -81,7 +80,7 @@ func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Ex
// Component backup tasks
logging.LogOperation(func() error {
st := status.NewWithCompat(ios.Stdout, 0)
st := status.NewWithCompat(progress, 0)
st.Start()
defer st.Stop()
@ -96,7 +95,7 @@ func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Ex
component.NewStagingContext(
ctx,
exporter.Environment,
stream.NewIOStream(writer, writer, nil, 0),
writer,
filepath.Join(backup.Description.Dest, bc.BackupName()),
manifest,
),
@ -111,11 +110,11 @@ func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Ex
}
return nil
}, ios, "Backing up core components")
}, progress, "Backing up core components")
// backup instances
logging.LogOperation(func() error {
st := status.NewWithCompat(ios.Stdout, 0)
st := status.NewWithCompat(progress, 0)
st.Start()
defer st.Stop()
@ -149,7 +148,7 @@ func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Ex
manifest <- dir
return exporter.NewSnapshot(ctx, instance, stream.NewIOStream(writer, writer, nil, 0), SnapshotDescription{
return exporter.NewSnapshot(ctx, instance, writer, SnapshotDescription{
Dest: dir,
})
},
@ -166,6 +165,6 @@ func (backup *Backup) run(ctx context.Context, ios stream.IOStream, exporter *Ex
})
return nil
}, ios, "Creating instance snapshots")
}, progress, "Creating instance snapshots")
}

34
internal/dis/component/exporter/iface.go
View file

@ -2,6 +2,7 @@ package exporter
import (
"context"
"fmt"
"io"
"path/filepath"
@ -11,7 +12,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/FAU-CDI/wisski-distillery/pkg/targz"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
)
// ExportTask describes a task that makes either a [Backup] or a [Snapshot].
@ -43,7 +43,7 @@ type export interface {
// MakeExport performs an export task as described by flags.
// Output is directed to the provided io.
func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, task ExportTask) (err error) {
func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, task ExportTask) (err error) {
// extract parameters
Title := "Backup"
Slug := ""
@ -53,7 +53,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
}
// determine target paths
logging.LogMessage(io, "Determining target paths")
logging.LogMessage(progress, "Determining target paths")
var stagingDir, archivePath string
if task.StagingOnly {
stagingDir = task.Dest
@ -69,11 +69,11 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
if !task.StagingOnly && archivePath == "" {
archivePath = exporter.NewArchivePath(Slug)
}
io.Printf("Staging Directory: %s\n", stagingDir)
io.Printf("Archive Path: %s\n", archivePath)
fmt.Fprintf(progress, "Staging Directory: %s\n", stagingDir)
fmt.Fprintf(progress, "Archive Path: %s\n", archivePath)
// create the staging directory
logging.LogMessage(io, "Creating staging directory")
logging.LogMessage(progress, "Creating staging directory")
err = exporter.Environment.Mkdir(stagingDir, environment.DefaultDirPerm)
if !environment.IsExist(err) && err != nil {
return err
@ -83,7 +83,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
// we need the staging directory to be deleted at the end
if !task.StagingOnly {
defer func() {
logging.LogMessage(io, "Removing staging directory")
logging.LogMessage(progress, "Removing staging directory")
exporter.Environment.RemoveAll(stagingDir)
}()
}
@ -96,11 +96,11 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
var sl export
if task.Instance == nil {
task.BackupDescription.Dest = stagingDir
backup := exporter.NewBackup(ctx, io, task.BackupDescription)
backup := exporter.NewBackup(ctx, progress, task.BackupDescription)
sl = &backup
} else {
task.SnapshotDescription.Dest = stagingDir
snapshot := exporter.NewSnapshot(ctx, task.Instance, io, task.SnapshotDescription)
snapshot := exporter.NewSnapshot(ctx, task.Instance, progress, task.SnapshotDescription)
sl = &snapshot
}
@ -109,7 +109,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
// find the report path
reportPath := filepath.Join(stagingDir, "report.txt")
io.Println(reportPath)
fmt.Fprintln(progress, reportPath)
// create the path
report, err := exporter.Environment.Create(reportPath, environment.DefaultFilePerm)
@ -122,28 +122,28 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
_, err := sl.Report(report)
return err
}
}, io, "Generating %s", Title)
}, progress, "Generating %s", Title)
// if we only requested staging
// all that is left is to write the log entry
if task.StagingOnly {
logging.LogMessage(io, "Writing Log Entry")
logging.LogMessage(progress, "Writing Log Entry")
// write out the log entry
entry.Path = stagingDir
entry.Packed = false
exporter.ExporterLogger.Add(ctx, entry)
io.Printf("Wrote %s\n", stagingDir)
fmt.Fprintf(progress, "Wrote %s\n", stagingDir)
return nil
}
// package everything up as an archive!
if err := logging.LogOperation(func() error {
var count int64
defer func() { io.Printf("Wrote %d byte(s) to %s\n", count, archivePath) }()
defer func() { fmt.Fprintf(progress, "Wrote %d byte(s) to %s\n", count, archivePath) }()
st := status.NewWithCompat(io.Stdout, 1)
st := status.NewWithCompat(progress, 1)
st.Start()
defer st.Stop()
@ -152,12 +152,12 @@ func (exporter *Exporter) MakeExport(ctx context.Context, io stream.IOStream, ta
})
return err
}, io, "Writing archive"); err != nil {
}, progress, "Writing archive"); err != nil {
return err
}
// write out the log entry
logging.LogMessage(io, "Writing Log Entry")
logging.LogMessage(progress, "Writing Log Entry")
entry.Path = archivePath
entry.Packed = true
exporter.ExporterLogger.Add(ctx, entry)

8
internal/dis/component/exporter/prune.go
View file

@ -2,10 +2,10 @@ package exporter
import (
"context"
"fmt"
"io"
"path/filepath"
"time"
"github.com/tkw1536/goprogram/stream"
)
// ShouldPrune determines if a file with the provided modification time should be
@ -15,7 +15,7 @@ func (exporter *Exporter) ShouldPrune(modtime time.Time) bool {
}
// Prune prunes all old exports
func (exporter *Exporter) PruneExports(ctx context.Context, io stream.IOStream) error {
func (exporter *Exporter) PruneExports(ctx context.Context, progress io.Writer) error {
sPath := exporter.ArchivePath()
// list all the files
@ -43,7 +43,7 @@ func (exporter *Exporter) PruneExports(ctx context.Context, io stream.IOStream)
// assemble path, and then remove the file!
path := filepath.Join(sPath, entry.Name())
io.Printf("Removing %s cause it is older than %d days\n", path, exporter.Config.MaxBackupAge)
fmt.Fprintf(progress, "Removing %s cause it is older than %d days\n", path, exporter.Config.MaxBackupAge)
if err := exporter.Still.Environment.Remove(path); err != nil {
return err

31
internal/dis/component/exporter/snapshot.go
View file

@ -14,7 +14,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/status"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
)
@ -45,20 +44,20 @@ type Snapshot struct {
}
// Snapshot creates a new snapshot of this instance into dest
func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, io stream.IOStream, desc SnapshotDescription) (snapshot Snapshot) {
func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.WissKI, progress io.Writer, desc SnapshotDescription) (snapshot Snapshot) {
logging.LogMessage(io, "Locking instance")
logging.LogMessage(progress, "Locking instance")
if !instance.Locker().TryLock(ctx) {
err := locker.Locked
io.EPrintln(err)
logging.LogMessage(io, "Aborting snapshot creation")
fmt.Fprintln(progress, err)
logging.LogMessage(progress, "Aborting snapshot creation")
return Snapshot{
ErrPanic: err,
}
}
defer func() {
logging.LogMessage(io, "Unlocking instance")
logging.LogMessage(progress, "Unlocking instance")
instance.Locker().Unlock(ctx)
}()
@ -75,27 +74,27 @@ func (snapshots *Exporter) NewSnapshot(ctx context.Context, instance *wisski.Wis
logging.LogOperation(func() error {
snapshot.StartTime = time.Now().UTC()
snapshot.ErrWhitebox = snapshot.makeParts(ctx, io, snapshots, instance, false)
snapshot.ErrBlackbox = snapshot.makeParts(ctx, io, snapshots, instance, true)
snapshot.ErrWhitebox = snapshot.makeParts(ctx, progress, snapshots, instance, false)
snapshot.ErrBlackbox = snapshot.makeParts(ctx, progress, snapshots, instance, true)
snapshot.EndTime = time.Now().UTC()
return nil
}, io, "Writing snapshot files")
}, progress, "Writing snapshot files")
slices.Sort(snapshot.Manifest)
return
}
func (snapshot *Snapshot) makeParts(ctx context.Context, ios stream.IOStream, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) map[string]error {
func (snapshot *Snapshot) makeParts(ctx context.Context, progress io.Writer, snapshots *Exporter, instance *wisski.WissKI, needsRunning bool) map[string]error {
if !needsRunning && !snapshot.Description.Keepalive {
stack := instance.Barrel().Stack()
logging.LogMessage(ios, "Stopping instance")
snapshot.ErrStop = stack.Down(ctx, ios)
logging.LogMessage(progress, "Stopping instance")
snapshot.ErrStop = stack.Down(ctx, progress)
defer func() {
logging.LogMessage(ios, "Starting instance")
snapshot.ErrStart = stack.Up(ctx, ios)
logging.LogMessage(progress, "Starting instance")
snapshot.ErrStart = stack.Up(ctx, progress)
}()
}
// handle writing the manifest!
@ -103,7 +102,7 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, ios stream.IOStream, sn
defer done()
// create a new status
st := status.NewWithCompat(ios.Stdout, 0)
st := status.NewWithCompat(progress, 0)
st.Start()
defer st.Stop()
@ -126,7 +125,7 @@ func (snapshot *Snapshot) makeParts(ctx context.Context, ios stream.IOStream, sn
component.NewStagingContext(
ctx,
snapshots.Environment,
stream.NewIOStream(writer, writer, nil, 0),
writer,
filepath.Join(snapshot.Description.Dest, sc.SnapshotName()),
manifest,
),

6
internal/dis/component/installable.go
View file

@ -2,9 +2,9 @@ package component
import (
"context"
"io"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/tkw1536/goprogram/stream"
)
// Installable implements an installable component.
@ -39,8 +39,8 @@ type Updatable interface {
// Update updates or initializes the provided components.
// It is called after the component has been installed (if applicable).
//
// It may send output to the provided stream.
// It may send progress to the provided stream.
//
// Updating should be idempotent, meaning running it multiple times must not break the existing system.
Update(ctx context.Context, stream stream.IOStream) error
Update(ctx context.Context, progress io.Writer) error
}

7
internal/dis/component/instances/runtime.go
View file

@ -3,10 +3,11 @@ package instances
import (
"context"
"embed"
"fmt"
"io"
"github.com/FAU-CDI/wisski-distillery/pkg/unpack"
"github.com/tkw1536/goprogram/exit"
"github.com/tkw1536/goprogram/stream"
)
var errBootstrapFailedRuntime = exit.Error{
@ -20,9 +21,9 @@ var errBootstrapFailedRuntime = exit.Error{
var runtimeResources embed.FS
// Update installs or updates runtime components needed by this component.
func (instances *Instances) Update(ctx context.Context, stream stream.IOStream) error {
func (instances *Instances) Update(ctx context.Context, progress io.Writer) error {
err := unpack.InstallDir(instances.Still.Environment, instances.Config.RuntimeDir(), "runtime", runtimeResources, func(dst, src string) {
stream.Printf("[copy] %s\n", dst)
fmt.Fprintf(progress, "[copy] %s\n", dst)
})
if err != nil {
return errBootstrapFailedRuntime.Wrap(err)

9
internal/dis/component/resolver/prefixes.go
View file

@ -2,17 +2,18 @@ package resolver
import (
"context"
"fmt"
"io"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/timex"
"github.com/tkw1536/goprogram/stream"
)
// updatePrefixes starts updating prefixes
func (resolver *Resolver) updatePrefixes(ctx context.Context, io stream.IOStream) {
func (resolver *Resolver) updatePrefixes(ctx context.Context, progress io.Writer) {
go func() {
for t := range timex.TickContext(ctx, resolver.RefreshInterval) {
io.Printf("[%s]: reloading prefixes\n", t.Format(time.Stamp))
fmt.Fprintf(progress, "[%s]: reloading prefixes\n", t.Format(time.Stamp))
err := (func() (err error) {
ctx, cancel := context.WithTimeout(ctx, resolver.RefreshInterval)
@ -27,7 +28,7 @@ func (resolver *Resolver) updatePrefixes(ctx context.Context, io stream.IOStream
return nil
})()
if err != nil {
io.EPrintf("error reloading prefixes: ", err.Error())
fmt.Fprintf(progress, "error reloading prefixes: %s", err.Error())
}
}
}()

10
internal/dis/component/resolver/resolver.go
View file

@ -3,6 +3,7 @@ package resolver
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"time"
@ -12,7 +13,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/instances"
"github.com/FAU-CDI/wisski-distillery/pkg/lazy"
"github.com/tkw1536/goprogram/stream"
)
type Resolver struct {
@ -32,7 +32,7 @@ var (
func (resolver *Resolver) Routes() []string { return []string{"/go/", "/wisski/get/"} }
func (resolver *Resolver) Handler(ctx context.Context, route string, io stream.IOStream) (http.Handler, error) {
func (resolver *Resolver) Handler(ctx context.Context, route string, progress io.Writer) (http.Handler, error) {
var err error
return resolver.handler.Get(func() (p wdresolve.ResolveHandler) {
p.TrustXForwardedProto = true
@ -45,17 +45,17 @@ func (resolver *Resolver) Handler(ctx context.Context, route string, io stream.I
domainName := resolver.Config.DefaultDomain
if domainName != "" {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domainName))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering default domain %s\n", domainName)
fmt.Fprintf(progress, "registering default domain %s\n", domainName)
}
// handle the extra domains!
for _, domain := range resolver.Config.SelfExtraDomains {
fallback.Data[fmt.Sprintf("^https?://(.*)\\.%s", regexp.QuoteMeta(domain))] = fmt.Sprintf("https://$1.%s", domainName)
io.Printf("registering legacy domain %s\n", domain)
fmt.Fprintf(progress, "registering legacy domain %s\n", domain)
}
// start updating prefixes
resolver.updatePrefixes(ctx, io)
resolver.updatePrefixes(ctx, progress)
// resolve the prefixes
p.Resolver = resolvers.InOrder{

5
internal/dis/component/server.go
View file

@ -2,9 +2,8 @@ package component
import (
"context"
"io"
"net/http"
"github.com/tkw1536/goprogram/stream"
)
// Servable is a component that is servable
@ -15,5 +14,5 @@ type Servable interface {
Routes() []string
// Handler returns the handler for the requested route
Handler(ctx context.Context, route string, io stream.IOStream) (http.Handler, error)
Handler(ctx context.Context, route string, progress io.Writer) (http.Handler, error)
}

4
internal/dis/component/sql/backup.go
View file

@ -6,6 +6,7 @@ import (
"io"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
"github.com/tkw1536/goprogram/stream"
)
var errSQLBackup = errors.New("SQLBackup: Mysqldump returned non-zero exit code")
@ -17,8 +18,7 @@ func (*SQL) BackupName() string {
// Backup makes a backup of all SQL databases into the path dest.
func (sql *SQL) Backup(scontext component.StagingContext) error {
return scontext.AddFile("", func(ctx context.Context, file io.Writer) error {
io := scontext.IO().Streams(file, nil, nil, 0).NonInteractive()
code, err := sql.Stack(sql.Environment).Exec(ctx, io, "sql", "mysqldump", "--all-databases")
code, err := sql.Stack(sql.Environment).Exec(ctx, stream.NonInteractive(scontext.Progress()), "sql", "mysqldump", "--all-databases")
if err != nil {
return err
}

8
internal/dis/component/sql/snapshot.go
View file

@ -16,16 +16,14 @@ func (*SQL) SnapshotName() string { return "sql" }
func (sql *SQL) Snapshot(wisski models.Instance, scontext component.StagingContext) error {
return scontext.AddDirectory(".", func(ctx context.Context) error {
return scontext.AddFile(wisski.SqlDatabase+".sql", func(ctx context.Context, file io.Writer) error {
return sql.SnapshotDB(ctx, scontext.IO(), file, wisski.SqlDatabase)
return sql.SnapshotDB(ctx, scontext.Progress(), file, wisski.SqlDatabase)
})
})
}
// SnapshotDB makes a backup of the sql database into dest.
func (sql *SQL) SnapshotDB(ctx context.Context, io stream.IOStream, dest io.Writer, database string) error {
io = io.Streams(dest, nil, nil, 0).NonInteractive()
code, err := sql.Stack(sql.Environment).Exec(ctx, io, "sql", "mysqldump", "--databases", database)
func (sql *SQL) SnapshotDB(ctx context.Context, progress io.Writer, dest io.Writer, database string) error {
code, err := sql.Stack(sql.Environment).Exec(ctx, stream.NonInteractive(progress), "sql", "mysqldump", "--databases", database)
if err != nil {
return err
}

13
internal/dis/component/sql/update.go
View file

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/models"
@ -44,14 +45,14 @@ var errSQLUnableToMigrate = exit.Error{
}
// Update initializes or updates the SQL database.
func (sql *SQL) Update(ctx context.Context, io stream.IOStream) error {
func (sql *SQL) Update(ctx context.Context, progress io.Writer) error {
// unsafely create the admin user!
{
if err := sql.unsafeWaitShell(ctx); err != nil {
return err
}
logging.LogMessage(io, "Creating administrative user")
logging.LogMessage(progress, "Creating administrative user")
{
username := sql.Config.MysqlAdminUser
password := sql.Config.MysqlAdminPassword
@ -62,7 +63,7 @@ func (sql *SQL) Update(ctx context.Context, io stream.IOStream) error {
}
// create the admin user
logging.LogMessage(io, "Creating sql database")
logging.LogMessage(progress, "Creating sql database")
{
if !sqle.IsSafeDatabaseLiteral(sql.Config.DistilleryDatabase) {
return errSQLUnsafeDatabaseName
@ -74,7 +75,7 @@ func (sql *SQL) Update(ctx context.Context, io stream.IOStream) error {
}
// wait for the database to come up
logging.LogMessage(io, "Waiting for database update to be complete")
logging.LogMessage(progress, "Waiting for database update to be complete")
sql.WaitQueryTable(ctx)
tables := []struct {
@ -107,7 +108,7 @@ func (sql *SQL) Update(ctx context.Context, io stream.IOStream) error {
// migrate all of the tables!
return logging.LogOperation(func() error {
for _, table := range tables {
logging.LogMessage(io, "migrating %q table", table.name)
logging.LogMessage(progress, "migrating %q table", table.name)
db, err := sql.QueryTable(ctx, false, table.table)
if err != nil {
return errSQLUnableToMigrate.WithMessageF(table.name, "unable to access table")
@ -118,5 +119,5 @@ func (sql *SQL) Update(ctx context.Context, io stream.IOStream) error {
}
}
return nil
}, io, "migrating database tables")
}, progress, "migrating database tables")
}

6
internal/dis/component/ssh2/server.go
View file

@ -2,9 +2,9 @@ package ssh2
import (
"context"
"io"
"github.com/gliderlabs/ssh"
"github.com/tkw1536/goprogram/stream"
)
const (
@ -13,10 +13,10 @@ const (
)
// Server returns an ssh server that implements the main ssh server
func (ssh2 *SSH2) Server(context context.Context, privateKeyPath string, io stream.IOStream) (*ssh.Server, error) {
func (ssh2 *SSH2) Server(context context.Context, privateKeyPath string, progress io.Writer) (*ssh.Server, error) {
var server ssh.Server
if err := ssh2.setupHostKeys(io, privateKeyPath, &server); err != nil {
if err := ssh2.setupHostKeys(progress, privateKeyPath, &server); err != nil {
return nil, err
}

28
internal/dis/component/ssh2/server_hostkeys.go
View file

@ -6,32 +6,32 @@ import (
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"io"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/gliderlabs/ssh"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
gossh "golang.org/x/crypto/ssh"
)
func (ssh2 *SSH2) setupHostKeys(io stream.IOStream, privateKeyPath string, server *ssh.Server) error {
return ssh2.UseOrMakeHostKeys(io, server, privateKeyPath, nil)
func (ssh2 *SSH2) setupHostKeys(progress io.Writer, privateKeyPath string, server *ssh.Server) error {
return ssh2.UseOrMakeHostKeys(progress, server, privateKeyPath, nil)
}
// UseOrMakeHostKeys is like UseOrMakeHostKey except that it accepts multiple HostKeyAlgorithms.
// For each key algorithm, the privateKeyPath is appended with "_" + the name of the algorithm in question.
//
// When algorithms is nil, picks a reasonable set of default algorithms.
func (ssh2 *SSH2) UseOrMakeHostKeys(io stream.IOStream, server *ssh.Server, privateKeyPath string, algorithms []HostKeyAlgorithm) error {
func (ssh2 *SSH2) UseOrMakeHostKeys(progress io.Writer, server *ssh.Server, privateKeyPath string, algorithms []HostKeyAlgorithm) error {
if algorithms == nil {
algorithms = []HostKeyAlgorithm{RSAAlgorithm, ED25519Algorithm}
}
for _, algorithm := range algorithms {
path := privateKeyPath + "_" + string(algorithm)
if err := ssh2.UseOrMakeHostKey(io, server, path, algorithm); err != nil {
if err := ssh2.UseOrMakeHostKey(progress, server, path, algorithm); err != nil {
return err
}
}
@ -44,8 +44,8 @@ func (ssh2 *SSH2) UseOrMakeHostKeys(io stream.IOStream, server *ssh.Server, priv
//
// All parameters except the server are passed to ReadOrMakeHostKey.
// Please see the appropriate documentation for that function.
func (ssh2 *SSH2) UseOrMakeHostKey(io stream.IOStream, server *ssh.Server, privateKeyPath string, algorithm HostKeyAlgorithm) error {
key, err := ssh2.ReadOrMakeHostKey(io, privateKeyPath, algorithm)
func (ssh2 *SSH2) UseOrMakeHostKey(progress io.Writer, server *ssh.Server, privateKeyPath string, algorithm HostKeyAlgorithm) error {
key, err := ssh2.ReadOrMakeHostKey(progress, privateKeyPath, algorithm)
if err != nil {
return err
}
@ -60,17 +60,17 @@ func (ssh2 *SSH2) UseOrMakeHostKey(io stream.IOStream, server *ssh.Server, priva
//
// This function assumes that if there is a host key in privateKeyPath it uses the provided HostKeyAlgorithm.
// It makes no attempt at verifiying this; the key mail fail to load and return an error, or it may load incorrect data.
func (ssh2 *SSH2) ReadOrMakeHostKey(io stream.IOStream, privateKeyPath string, algorithm HostKeyAlgorithm) (key gossh.Signer, err error) {
func (ssh2 *SSH2) ReadOrMakeHostKey(progress io.Writer, privateKeyPath string, algorithm HostKeyAlgorithm) (key gossh.Signer, err error) {
hostKey := NewHostKey(algorithm)
if _, e := ssh2.Environment.Lstat(privateKeyPath); environment.IsNotExist(e) { // path doesn't exist => generate a new key there!
err = ssh2.makeHostKey(io, hostKey, privateKeyPath)
err = ssh2.makeHostKey(progress, hostKey, privateKeyPath)
if err != nil {
err = errors.Wrap(err, "Unable to generate new host key")
return
}
}
err = ssh2.loadHostKey(io, hostKey, privateKeyPath)
err = ssh2.loadHostKey(progress, hostKey, privateKeyPath)
if err != nil {
return nil, err
}
@ -78,8 +78,8 @@ func (ssh2 *SSH2) ReadOrMakeHostKey(io stream.IOStream, privateKeyPath string, a
}
// loadHostKey loadsa host key
func (ssh2 *SSH2) loadHostKey(io stream.IOStream, key HostKey, path string) (err error) {
io.EPrintf("Loading hostkey (algorithm %s) from %q", key.Algorithm(), path)
func (ssh2 *SSH2) loadHostKey(progress io.Writer, key HostKey, path string) (err error) {
fmt.Fprintf(progress, "Loading hostkey (algorithm %s) from %q", key.Algorithm(), path)
// read all the bytes from the file
privateKeyBytes, err := environment.ReadFile(ssh2.Environment, path)
@ -104,8 +104,8 @@ func (ssh2 *SSH2) loadHostKey(io stream.IOStream, key HostKey, path string) (err
}
// makeHostKey makes a new host key
func (ssh2 *SSH2) makeHostKey(io stream.IOStream, key HostKey, path string) error {
io.EPrintf("Writing hostkey (algorithm %s) to %q", key.Algorithm(), path)
func (ssh2 *SSH2) makeHostKey(progress io.Writer, key HostKey, path string) error {
fmt.Fprintf(progress, "Writing hostkey (algorithm %s) to %q", key.Algorithm(), path)
if err := key.Generate(0, nil); err != nil {
return errors.Wrap(err, "Failed to generate key")

38
internal/dis/component/stack.go
View file

@ -5,6 +5,8 @@ import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/fs"
"path/filepath"
@ -34,9 +36,9 @@ var errStackUpdateBuild = errors.New("Stack.Update: Build returned non-zero exit
// This does not have a direct 'docker compose' shell equivalent.
//
// See also Up.
func (ds Stack) Update(ctx context.Context, io stream.IOStream, start bool) error {
func (ds Stack) Update(ctx context.Context, progress io.Writer, start bool) error {
{
code, err := ds.compose(ctx, io, "pull")
code, err := ds.compose(ctx, stream.NonInteractive(progress), "pull")
if err != nil {
return err
}
@ -46,7 +48,7 @@ func (ds Stack) Update(ctx context.Context, io stream.IOStream, start bool) erro
}
{
code, err := ds.compose(ctx, io, "build", "--pull")
code, err := ds.compose(ctx, stream.NonInteractive(progress), "build", "--pull")
if err != nil {
return err
}
@ -55,7 +57,7 @@ func (ds Stack) Update(ctx context.Context, io stream.IOStream, start bool) erro
}
}
if start {
return ds.Up(ctx, io)
return ds.Up(ctx, progress)
}
return nil
}
@ -64,8 +66,8 @@ var errStackUp = errors.New("Stack.Up: Up returned non-zero exit code")
// Up creates and starts the containers in this Stack.
// It is equivalent to 'docker compose up --remove-orphans --detach' on the shell.
func (ds Stack) Up(ctx context.Context, io stream.IOStream) error {
code, err := ds.compose(ctx, io, "up", "--remove-orphans", "--detach")
func (ds Stack) Up(ctx context.Context, progress io.Writer) error {
code, err := ds.compose(ctx, stream.NonInteractive(progress), "up", "--remove-orphans", "--detach")
if err != nil {
return err
}
@ -116,8 +118,8 @@ var errStackRestart = errors.New("Stack.Restart: Restart returned non-zero exit
// Restart restarts all containers in this Stack.
// It is equivalent to 'docker compose restart' on the shell.
func (ds Stack) Restart(ctx context.Context, io stream.IOStream) error {
code, err := ds.compose(ctx, io, "restart")
func (ds Stack) Restart(ctx context.Context, progress io.Writer) error {
code, err := ds.compose(ctx, stream.NonInteractive(progress), "restart")
if err != nil {
return err
}
@ -130,12 +132,12 @@ func (ds Stack) Restart(ctx context.Context, io stream.IOStream) error {
var errStackPs = errors.New("Stack.Ps: Down returned non-zero exit code")
// Ps returns the ids of the containers currently running
func (ds Stack) Ps(ctx context.Context, io stream.IOStream) ([]string, error) {
func (ds Stack) Ps(ctx context.Context, progress io.Writer) ([]string, error) {
// create a buffer
var buffer bytes.Buffer
// read the ids from the command!
code, err := ds.compose(ctx, io.Streams(&buffer, nil, nil, 0), "ps", "-q")
code, err := ds.compose(ctx, stream.NewIOStream(&buffer, progress, nil, 0), "ps", "-q")
if err != nil {
return nil, err
}
@ -163,8 +165,8 @@ var errStackDown = errors.New("Stack.Down: Down returned non-zero exit code")
// Down stops and removes all containers in this Stack.
// It is equivalent to 'docker compose down -v' on the shell.
func (ds Stack) Down(ctx context.Context, io stream.IOStream) error {
code, err := ds.compose(ctx, io, "down", "-v")
func (ds Stack) Down(ctx context.Context, progress io.Writer) error {
code, err := ds.compose(ctx, stream.NonInteractive(progress), "down", "-v")
if err != nil {
return err
}
@ -219,7 +221,7 @@ type InstallationContext map[string]string
//
// Installation is non-interactive, but will provide debugging output onto io.
// InstallationContext
func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, context InstallationContext) error {
func (is StackWithResources) Install(ctx context.Context, progress io.Writer, context InstallationContext) error {
env := is.Stack.Env
if is.ContextPath != "" {
// setup the base files
@ -229,7 +231,7 @@ func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, co
is.ContextPath,
is.Resources,
func(dst, src string) {
io.Printf("[install] %s\n", dst)
fmt.Fprintf(progress, "[install] %s\n", dst)
},
); err != nil {
return err
@ -239,7 +241,7 @@ func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, co
// configure .env
envDest := filepath.Join(is.Dir, ".env")
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
fmt.Fprintf(progress, "[config] %s\n", envDest)
if err := unpack.InstallTemplate(
env,
envDest,
@ -256,7 +258,7 @@ func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, co
// find the destination!
dst := filepath.Join(is.Dir, name)
io.Printf("[make] %s\n", dst)
fmt.Fprintf(progress, "[make] %s\n", dst)
if is.MakeDirsPerm == fs.FileMode(0) {
is.MakeDirsPerm = environment.DefaultDirPerm
}
@ -277,7 +279,7 @@ func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, co
dst := filepath.Join(is.Dir, name)
// copy over file from context
io.Printf("[copy] %s (from %s)\n", dst, src)
fmt.Fprintf(progress, "[copy] %s (from %s)\n", dst, src)
if err := fsx.CopyFile(ctx, env, dst, src); err != nil {
return errors.Wrapf(err, "Unable to copy file %s", src)
}
@ -288,7 +290,7 @@ func (is StackWithResources) Install(ctx context.Context, io stream.IOStream, co
// find the destination!
dst := filepath.Join(is.Dir, name)
io.Printf("[touch] %s\n", dst)
fmt.Fprintf(progress, "[touch] %s\n", dst)
if err := fsx.Touch(env, dst, is.TouchFilesPerm); err != nil {
return err
}

12
internal/dis/component/triplestore/update.go
View file

@ -3,22 +3,22 @@ package triplestore
import (
"context"
"fmt"
"io"
"net/http"
"github.com/FAU-CDI/wisski-distillery/pkg/logging"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
var errTriplestoreFailedSecurity = errors.New("failed to enable triplestore security: request did not succeed with HTTP 200 OK")
func (ts Triplestore) Update(ctx context.Context, io stream.IOStream) error {
logging.LogMessage(io, "Waiting for Triplestore")
func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
logging.LogMessage(progress, "Waiting for Triplestore")
if err := ts.Wait(ctx); err != nil {
return err
}
logging.LogMessage(io, "Resetting admin user password")
logging.LogMessage(progress, "Resetting admin user password")
{
res, err := ts.OpenRaw(ctx, "PUT", "/rest/security/users/"+ts.Config.TriplestoreAdminUser, TriplestoreUserPayload{
Password: ts.Config.TriplestoreAdminPassword,
@ -43,14 +43,14 @@ func (ts Triplestore) Update(ctx context.Context, io stream.IOStream) error {
case http.StatusUnauthorized:
// a password is needed => security is already enabled.
// the password may or may not work, but that's a problem for later
logging.LogMessage(io, "Security is already enabled")
logging.LogMessage(progress, "Security is already enabled")
return nil
default:
return fmt.Errorf("failed to create triplestore user: %s", err)
}
}
logging.LogMessage(io, "Enabling Triplestore security")
logging.LogMessage(progress, "Enabling Triplestore security")
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/security", true, "", "")
if err != nil {

8
internal/wisski/ingredient/barrel/build.go
View file

@ -2,6 +2,7 @@ package barrel
import (
"context"
"io"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
@ -10,13 +11,12 @@ import (
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/locker"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/mstore"
"github.com/tkw1536/goprogram/stream"
)
// Build builds or rebuilds the barel connected to this instance.
//
// It also logs the current time into the metadata belonging to this instance.
func (barrel *Barrel) Build(ctx context.Context, stream stream.IOStream, start bool) error {
func (barrel *Barrel) Build(ctx context.Context, progress io.Writer, start bool) error {
if !barrel.Locker.TryLock(ctx) {
err := locker.Locked
return err
@ -28,14 +28,14 @@ func (barrel *Barrel) Build(ctx context.Context, stream stream.IOStream, start b
var context component.InstallationContext
{
err := stack.Install(ctx, stream, context)
err := stack.Install(ctx, progress, context)
if err != nil {
return err
}
}
{
err := stack.Update(ctx, stream, start)
err := stack.Update(ctx, progress, start)
if err != nil {
return err
}

11
internal/wisski/ingredient/barrel/drush/cron.go
View file

@ -2,8 +2,11 @@ package drush
import (
"context"
"fmt"
"time"
"io"
"github.com/FAU-CDI/wisski-distillery/internal/phpx"
"github.com/FAU-CDI/wisski-distillery/internal/status"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
@ -16,15 +19,15 @@ var errCronFailed = exit.Error{
ExitCode: exit.ExitGeneric,
}
func (drush *Drush) Cron(ctx context.Context, io stream.IOStream) error {
code, err := drush.Barrel.Shell(ctx, io, "/runtime/cron.sh")
func (drush *Drush) Cron(ctx context.Context, progress io.Writer) error {
code, err := drush.Barrel.Shell(ctx, stream.NonInteractive(progress), "/runtime/cron.sh")
if err != nil {
io.EPrintln(err)
fmt.Fprintln(progress, err)
}
if code != 0 {
// keep going, because we want to run as many crons as possible
err = errCronFailed.WithMessageF(drush.Slug, code)
io.EPrintln(err)
fmt.Fprintln(progress, err)
}
return nil

5
internal/wisski/ingredient/barrel/drush/update.go
View file

@ -2,6 +2,7 @@ package drush
import (
"context"
"io"
"time"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component/meta"
@ -19,8 +20,8 @@ var errBlindUpdateFailed = exit.Error{
}
// Update performs a blind drush update
func (drush *Drush) Update(ctx context.Context, io stream.IOStream) error {
code, err := drush.Barrel.Shell(ctx, io, "/runtime/blind_update.sh")
func (drush *Drush) Update(ctx context.Context, progress io.Writer) error {
code, err := drush.Barrel.Shell(ctx, stream.NonInteractive(progress), "/runtime/blind_update.sh")
if err != nil {
return errBlindUpdateFailed.WithMessageF(drush.Slug, environment.ExecCommandError)
}

7
internal/wisski/ingredient/barrel/provisioner/provisioner.go
View file

@ -3,6 +3,7 @@ package provisioner
import (
"context"
"errors"
"io"
"strings"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
@ -20,10 +21,10 @@ type Provisioner struct {
}
// Provision provisions an instance, assuming that the required databases already exist.
func (provision *Provisioner) Provision(ctx context.Context, io stream.IOStream) error {
func (provision *Provisioner) Provision(ctx context.Context, progress io.Writer) error {
// build the container
if err := provision.Barrel.Build(ctx, io, false); err != nil {
if err := provision.Barrel.Build(ctx, progress, false); err != nil {
return err
}
@ -54,7 +55,7 @@ func (provision *Provisioner) Provision(ctx context.Context, io stream.IOStream)
// TODO: Move the provision script into the control plane!
provisionScript := "sudo PATH=$PATH -u www-data /bin/bash /provision_container.sh " + strings.Join(provisionParams, " ")
code, err := provision.Barrel.Stack().Run(ctx, io, true, "barrel", "/bin/bash", "-c", provisionScript)
code, err := provision.Barrel.Stack().Run(ctx, stream.NonInteractive(progress), true, "barrel", "/bin/bash", "-c", provisionScript)
if err != nil {
return err
}

8
internal/wisski/ingredient/barrel/running.go
View file

@ -2,15 +2,15 @@ package barrel
import (
"context"
"io"
"github.com/FAU-CDI/wisski-distillery/internal/status"
"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
"github.com/tkw1536/goprogram/stream"
)
// Running checks if this WissKI is currently running.
func (barrel *Barrel) Running(ctx context.Context) (bool, error) {
ps, err := barrel.Stack().Ps(ctx, stream.FromNil())
func (barrel *Barrel) Running(ctx context.Context, progress io.Writer) (bool, error) {
ps, err := barrel.Stack().Ps(ctx, progress)
if err != nil {
return false, err
}
@ -28,6 +28,6 @@ var (
)
func (rf *RunningFetcher) Fetch(flags ingredient.FetcherFlags, info *status.WissKI) (err error) {
info.Running, err = rf.Barrel.Running(flags.Context)
info.Running, err = rf.Barrel.Running(flags.Context, io.Discard)
return
}

13
pkg/httpx/socket.go
View file

@ -89,20 +89,21 @@ func (h *WebSocket) serveWebsocket(w http.ResponseWriter, r *http.Request) {
h.pool.Get(nil).Put(socket)
}
// WebSocketConnection represents a connected Websocket
// WebSocketConnection represents a connected WebSocket.
type WebSocketConnection interface {
// Context returns a context that is closed once this websocket is closed.
// Context returns a context that is closed once the connection is terminated.
Context() context.Context
// Read returns a channel that receives message.
// The channel is closed once no more messags are available.
// The channel is closed if no more messags are available (for instance because the server closed).
Read() <-chan WebSocketMessage
// Write queues the provided message for sending
// and returns a channel that is closed once the message has been sent.
// Write queues the provided message for sending.
// The returned channel is closed once the message has been sent.
Write(WebSocketMessage) <-chan struct{}
// WriteText is a convenience method to send a TextMessage
// WriteText is a convenience method to send a TextMessage.
// The returned channel is closed once the message has been sent.
WriteText(text string) <-chan struct{}
// Close closes the underlying connection

123
pkg/logging/level.go
View file

@ -1,57 +1,108 @@
package logging
import (
"io"
"sync"
"github.com/tkw1536/goprogram/stream"
)
var logLevelMutex sync.Mutex
var logLevelMap = make(map[uintptr]int)
type writerIndent struct{}
func getIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
var indentKey = writerIndent{}
id, ok := logID(io)
func getIndent(writer io.Writer) int {
value, ok := getKey(writer, indentKey)
if !ok {
value = 0
}
return value.(int)
}
func incIndent(writer io.Writer) int {
value, ok := upsetKey(writer, indentKey, func(value any, fresh bool) any {
if fresh {
return 0
}
return value.(int) + 1
})
if !ok {
return 0
}
return logLevelMap[id]
return value.(int)
}
func incIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
id, ok := logID(io)
if !ok { // if we don't have an id, then inc statically returns 1
return 1
}
logLevelMap[id]++
return logLevelMap[id]
}
func decIndent(io stream.IOStream) int {
logLevelMutex.Lock()
defer logLevelMutex.Unlock()
id, ok := logID(io)
if !ok { // if we don't have an id, then dec statically returns 0
func decIndent(writer io.Writer) int {
value, ok := upsetKey(writer, indentKey, func(value any, fresh bool) any {
if fresh {
return 0
}
logLevelMap[id]--
if logLevelMap[id] < 0 {
panic("DecLogIdent: decrease below 0")
level := value.(int) - 1
if level < 0 {
level = 0
}
return logLevelMap[id]
return level
})
if !ok {
return 0
}
return value.(int)
}
func logID(io stream.IOStream) (uintptr, bool) {
file, ok := io.Stdin.(interface{ Fd() uintptr })
// KEY-VALUE STORE for writers
var writerDataMutex sync.RWMutex
var writerDataData = make(map[uintptr]map[any]any)
func getKey(writer io.Writer, key any) (value any, ok bool) {
uid, ok := id(writer)
if !ok {
return nil, false
}
writerDataMutex.RLock()
defer writerDataMutex.RUnlock()
value, ok = writerDataData[uid][key]
return
}
func setKey(writer io.Writer, key, value any) bool {
uid, ok := id(writer)
if !ok {
return false
}
writerDataMutex.Lock()
defer writerDataMutex.Unlock()
values, ok := writerDataData[uid]
if !ok {
values = make(map[any]any)
writerDataData[uid] = values
}
values[key] = value
return true
}
func upsetKey(writer io.Writer, key any, update func(value any, fresh bool) any) (any, bool) {
uid, ok := id(writer)
if !ok {
return nil, false
}
writerDataMutex.Lock()
defer writerDataMutex.Unlock()
values, ok := writerDataData[uid]
if !ok {
values = make(map[any]any)
writerDataData[uid] = values
}
values[key] = update(values[key], !ok)
return values[key], true
}
func id(writer io.Writer) (uintptr, bool) {
file, ok := writer.(interface{ Fd() uintptr })
if !ok {
return 0, false
}

28
pkg/logging/log.go
View file

@ -1,30 +1,38 @@
package logging
import (
"fmt"
"io"
"strings"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/term"
)
// LogOperation logs a message that is displayed to the user, and then increases the log indent level.
func LogOperation(operation func() error, io stream.IOStream, format string, args ...interface{}) error {
logOperation(io, getIndent(io), format, args...)
incIndent(io)
defer decIndent(io)
func LogOperation(operation func() error, progress io.Writer, format string, args ...interface{}) error {
logOperation(progress, getIndent(progress), format, args...)
incIndent(progress)
defer decIndent(progress)
return operation()
}
// LogMessage logs a message that is displayed to the user
func LogMessage(io stream.IOStream, format string, args ...interface{}) (int, error) {
return logOperation(io, getIndent(io), format, args...)
func LogMessage(progress io.Writer, format string, args ...interface{}) (int, error) {
return logOperation(progress, getIndent(progress), format, args...)
}
func logOperation(io stream.IOStream, indent int, format string, args ...interface{}) (int, error) {
func logOperation(progress io.Writer, indent int, format string, args ...interface{}) (int, error) {
message := "\033[1m" + strings.Repeat(" ", indent+1) + "=> " + format + "\033[0m\n"
if !io.StdoutIsATerminal() {
if !streamIsTerminal(progress) {
message = " => " + format + "\n"
}
return io.Printf(message, args...)
return fmt.Fprintf(progress, message, args...)
}
// streamIsTerminal checks if stream is a terminal
func streamIsTerminal(stream any) bool {
file, ok := stream.(interface{ Fd() uintptr })
return ok && term.IsTerminal(int(file.Fd()))
}