rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix regression in stagingContext

Browse source
This commit is contained in:
Tom 2023-05-30 11:25:01 +02:00
parent 3a893bbfb6
commit fcdb3a1d6a
14 changed files with 78 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -16,7 +16,7 @@ require (
github.com/pquerna/otp v1.4.0 github.com/pquerna/otp v1.4.0
github.com/rs/zerolog v1.29.1 github.com/rs/zerolog v1.29.1
github.com/tkw1536/goprogram v0.3.5 github.com/tkw1536/goprogram v0.3.5
github.com/tkw1536/pkglib v0.0.0-20230503121722-b0c615fc34ee github.com/tkw1536/pkglib v0.0.0-20230530085130-4f049f64e420
github.com/yuin/goldmark v1.5.4 github.com/yuin/goldmark v1.5.4
github.com/yuin/goldmark-meta v1.1.0 github.com/yuin/goldmark-meta v1.1.0
golang.org/x/crypto v0.8.0 golang.org/x/crypto v0.8.0

4
go.sum
View file

@ -114,8 +114,8 @@ github.com/tdewolff/parse v2.3.4+incompatible/go.mod h1:8oBwCsVmUkgHO8M5iCzSIDtp
github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM= github.com/tdewolff/test v1.0.7 h1:8Vs0142DmPFW/bQeHRP3MV19m1gvndjUb1sn8yy74LM=
github.com/tkw1536/goprogram v0.3.5 h1:S0axKo3R/vGa4zhYqYDKAZEPhAfwUSSeMtVwnAu4sNY= github.com/tkw1536/goprogram v0.3.5 h1:S0axKo3R/vGa4zhYqYDKAZEPhAfwUSSeMtVwnAu4sNY=
github.com/tkw1536/goprogram v0.3.5/go.mod h1:pYr4dMHOSVurbPQ4KTR0ett8XWNISbsRS6zlh9Nsxa8= github.com/tkw1536/goprogram v0.3.5/go.mod h1:pYr4dMHOSVurbPQ4KTR0ett8XWNISbsRS6zlh9Nsxa8=
github.com/tkw1536/pkglib v0.0.0-20230503121722-b0c615fc34ee h1:MJZzhO8Fq7WVNGjWLhS4cGhWWmK92BMgzDtLSaYeFx4= github.com/tkw1536/pkglib v0.0.0-20230530085130-4f049f64e420 h1:ZZ7Hhf4cMKXWbOfh6A1FWmhOgD8US0Tz9Ky2fPiaZ3I=
github.com/tkw1536/pkglib v0.0.0-20230503121722-b0c615fc34ee/go.mod h1:0A1B9Cc5+yJXR3eeB14CqD4dFSbEjjWRo5Pr9M3XYuI= github.com/tkw1536/pkglib v0.0.0-20230530085130-4f049f64e420/go.mod h1:0A1B9Cc5+yJXR3eeB14CqD4dFSbEjjWRo5Pr9M3XYuI=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=

106
internal/dis/component/backup.go
View file

@ -2,6 +2,7 @@ package component
import ( import (
"context" "context"
"fmt"
"io" "io"
"path/filepath" "path/filepath"
@ -18,7 +19,7 @@ type Backupable interface {
BackupName() string BackupName() string
// Backup backs up this component into the destination path path // Backup backs up this component into the destination path path
Backup(context StagingContext) error Backup(context *StagingContext) error
} }
// Snapshotable represents a component with a Snapshot method. // Snapshotable represents a component with a Snapshot method.
@ -32,40 +33,12 @@ type Snapshotable interface {
SnapshotName() string SnapshotName() string
// Snapshot snapshots a part of the instance // Snapshot snapshots a part of the instance
Snapshot(wisski models.Instance, context StagingContext) error Snapshot(wisski models.Instance, context *StagingContext) error
}
// StagingContext represents a context for [Backupable] and [Snapshotable]
type StagingContext interface {
// Progress returns a writer to write progress information to.
Progress() io.Writer
// Name creates a new directory inside the destination.
// Passing the empty path creates the destination as a directory.
//
// It then allows op to fill the file.
AddDirectory(path string, op func(context.Context) error) error
// CopyFile copies a file from src to dst.
CopyFile(dst, src string) error
// CopyDirectory copies a directory from src to dst.
CopyDirectory(dst, src string) error
// AddFile creates a new file at the provided path inside the destination.
// Passing the empty path creates the destination as a file.
//
// It then allows op to write to the file.
//
// The op function must not retain file.
// The underlying file does not need to be closed.
// AddFile will not return before op has returned.
AddFile(path string, op func(ctx context.Context, file io.Writer) error) error
} }
// NewStagingContext returns a new [StagingContext] // NewStagingContext returns a new [StagingContext]
func NewStagingContext(ctx context.Context, progress io.Writer, path string, manifest chan<- string) StagingContext { func NewStagingContext(ctx context.Context, progress io.Writer, path string, manifest chan<- string) *StagingContext {
return &stagingContext{ return &StagingContext{
ctx: ctx, ctx: ctx,
progress: progress, progress: progress,
path: path, path: path,
@ -73,33 +46,46 @@ func NewStagingContext(ctx context.Context, progress io.Writer, path string, man
} }
} }
// stagingContext implements [components.StagingContext] // StagingContext is a context used for [Backupable] and [Snapshotable]
type stagingContext struct { type StagingContext struct {
ctx context.Context ctx context.Context
progress io.Writer // writer to direct progress to progress io.Writer // writer to direct progress to
path string // path to send files to path string // path to send files to
manifest chan<- string // channel the manifest is sent to manifest chan<- string // channel the manifest is sent to
} }
func (bc *stagingContext) sendPath(path string) { func (bc *StagingContext) sendPath(path string) {
// resolve the path, or bail out! // ensure path is absolute!
// TODO: Use the relative path here! if !filepath.IsAbs(path) {
dst, err := bc.resolve(path) var err error
path, err = bc.resolve(path)
if err != nil { if err != nil {
fmt.Fprintf(bc.progress, "path resolve error: %s", err)
return return
} }
io.WriteString(bc.progress, dst+"\n")
bc.manifest <- dst
} }
func (bc *stagingContext) Progress() io.Writer { // use the relative path for logging
rel, err := bc.relativize(path)
if err == nil {
io.WriteString(bc.progress, rel+"\n")
}
// send the absolute path
bc.manifest <- path
}
// Progress returns a writer to write progress information to.
func (bc *StagingContext) Progress() io.Writer {
return bc.progress return bc.progress
} }
var errResolveAbsolute = errors.New("resolve: path must be relative") var (
errResolveAbsolute = errors.New("resolve: path must be relative")
errRelativeRelative = errors.New("relativize: path already relative")
)
func (bc *stagingContext) resolve(path string) (dest string, err error) { func (bc *StagingContext) resolve(path string) (dest string, err error) {
if path == "" { if path == "" {
return bc.path, nil return bc.path, nil
} }
@ -109,7 +95,19 @@ func (bc *stagingContext) resolve(path string) (dest string, err error) {
return filepath.Join(bc.path, path), nil return filepath.Join(bc.path, path), nil
} }
func (sc *stagingContext) AddDirectory(path string, op func(context.Context) error) error { func (bc *StagingContext) relativize(path string) (dest string, err error) {
if !filepath.IsAbs(path) {
return "", errRelativeRelative
}
return filepath.Rel(bc.path, path)
}
// AddDirectory creates a new directory inside the destination.
// Passing the empty path creates the destination as a directory.
//
// It then allows op to fill the file.
func (sc *StagingContext) AddDirectory(path string, op func(context.Context) error) error {
// check if we are already done // check if we are already done
if err, ok := sc.ctxdone(); ok { if err, ok := sc.ctxdone(); ok {
return err return err
@ -133,7 +131,8 @@ func (sc *stagingContext) AddDirectory(path string, op func(context.Context) err
return op(sc.ctx) return op(sc.ctx)
} }
func (sc *stagingContext) CopyFile(dst, src string) error { // CopyFile copies a file from src to dst.
func (sc *StagingContext) CopyFile(dst, src string) error {
if err, ok := sc.ctxdone(); ok { if err, ok := sc.ctxdone(); ok {
return err return err
} }
@ -146,7 +145,8 @@ func (sc *stagingContext) CopyFile(dst, src string) error {
return umaskfree.CopyFile(sc.ctx, dstPath, src) return umaskfree.CopyFile(sc.ctx, dstPath, src)
} }
func (sc *stagingContext) CopyDirectory(dst, src string) error { // CopyDirectory copies a directory from src to dst.
func (sc *StagingContext) CopyDirectory(dst, src string) error {
if err, ok := sc.ctxdone(); ok { if err, ok := sc.ctxdone(); ok {
return err return err
} }
@ -161,7 +161,15 @@ func (sc *stagingContext) CopyDirectory(dst, src string) error {
}) })
} }
func (sc *stagingContext) AddFile(path string, op func(ctx context.Context, file io.Writer) error) error { // AddFile creates a new file at the provided path inside the destination.
// Passing the empty path creates the destination as a file.
//
// It then allows op to write to the file.
//
// The op function must not retain file.
// The underlying file does not need to be closed.
// AddFile will not return before op has returned.
func (sc *StagingContext) AddFile(path string, op func(ctx context.Context, file io.Writer) error) error {
// check if we're already done // check if we're already done
if err, ok := sc.ctxdone(); ok { if err, ok := sc.ctxdone(); ok {
return err return err
@ -187,7 +195,7 @@ func (sc *stagingContext) AddFile(path string, op func(ctx context.Context, file
return op(sc.ctx, file) return op(sc.ctx, file)
} }
func (sc *stagingContext) ctxdone() (err error, done bool) { func (sc *StagingContext) ctxdone() (err error, done bool) {
err = sc.ctx.Err() err = sc.ctx.Err()
done = (err != nil) done = (err != nil)
return return

2
internal/dis/component/exporter/extras_bookkeeping.go
View file

@ -24,7 +24,7 @@ func (Bookkeeping) SnapshotNeedsRunning() bool { return false }
func (Bookkeeping) SnapshotName() string { return "bookkeeping.txt" } func (Bookkeeping) SnapshotName() string { return "bookkeeping.txt" }
// Snapshot creates a snapshot of this instance // Snapshot creates a snapshot of this instance
func (*Bookkeeping) Snapshot(wisski models.Instance, scontext component.StagingContext) error { func (*Bookkeeping) Snapshot(wisski models.Instance, scontext *component.StagingContext) error {
return scontext.AddFile(".", func(ctx context.Context, file io.Writer) error { return scontext.AddFile(".", func(ctx context.Context, file io.Writer) error {
_, err := fmt.Fprintf(file, "%#v\n", wisski) _, err := fmt.Fprintf(file, "%#v\n", wisski)
return err return err

2
internal/dis/component/exporter/extras_config.go
View file

@ -20,7 +20,7 @@ func (*Config) BackupName() string {
return "config" return "config"
} }
func (control *Config) Backup(scontext component.StagingContext) error { func (control *Config) Backup(scontext *component.StagingContext) error {
files := control.backupFiles() files := control.backupFiles()
return scontext.AddDirectory("", func(ctx context.Context) error { return scontext.AddDirectory("", func(ctx context.Context) error {

2
internal/dis/component/exporter/extras_filesystem.go
View file

@ -21,6 +21,6 @@ func (Filesystem) SnapshotNeedsRunning() bool { return false }
func (Filesystem) SnapshotName() string { return "data" } func (Filesystem) SnapshotName() string { return "data" }
// Snapshot creates a snapshot of this instance // Snapshot creates a snapshot of this instance
func (*Filesystem) Snapshot(wisski models.Instance, context component.StagingContext) error { func (*Filesystem) Snapshot(wisski models.Instance, context *component.StagingContext) error {
return context.CopyDirectory(".", wisski.FilesystemBase) return context.CopyDirectory(".", wisski.FilesystemBase)
} }

2
internal/dis/component/exporter/extras_pathbuilders.go
View file

@ -24,7 +24,7 @@ func (Pathbuilders) SnapshotNeedsRunning() bool { return true }
func (Pathbuilders) SnapshotName() string { return "pathbuilders" } func (Pathbuilders) SnapshotName() string { return "pathbuilders" }
func (pbs *Pathbuilders) Snapshot(wisski models.Instance, scontext component.StagingContext) error { func (pbs *Pathbuilders) Snapshot(wisski models.Instance, scontext *component.StagingContext) error {
return scontext.AddDirectory(".", func(ctx context.Context) error { return scontext.AddDirectory(".", func(ctx context.Context) error {
builders, err := pbs.Dependencies.Instances.Instance(ctx, wisski).Pathbuilder().GetAll(ctx, nil) builders, err := pbs.Dependencies.Instances.Instance(ctx, wisski).Pathbuilder().GetAll(ctx, nil)
if err != nil { if err != nil {

12
internal/dis/component/exporter/iface.go
View file

@ -117,19 +117,19 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
// and retain a log entry // and retain a log entry
var entry models.Export var entry models.Export
logging.LogOperation(func() error { logging.LogOperation(func() error {
var sl export var export export
if task.Instance == nil { if task.Instance == nil {
task.BackupDescription.Dest = stagingDir task.BackupDescription.Dest = stagingDir
backup := exporter.NewBackup(ctx, progress, task.BackupDescription) backup := exporter.NewBackup(ctx, progress, task.BackupDescription)
sl = &backup export = &backup
} else { } else {
task.SnapshotDescription.Dest = stagingDir task.SnapshotDescription.Dest = stagingDir
snapshot := exporter.NewSnapshot(ctx, task.Instance, progress, task.SnapshotDescription) snapshot := exporter.NewSnapshot(ctx, task.Instance, progress, task.SnapshotDescription)
sl = &snapshot export = &snapshot
} }
// create a log entry // create a log entry
entry = sl.LogEntry() entry = export.LogEntry()
// write the machine report // write the machine report
{ {
@ -141,7 +141,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
return err return err
} }
if err := sl.ReportMachine(report); err != nil { if err := export.ReportMachine(report); err != nil {
return err return err
} }
} }
@ -156,7 +156,7 @@ func (exporter *Exporter) MakeExport(ctx context.Context, progress io.Writer, ta
return err return err
} }
if err := sl.ReportPlain(report); err != nil { if err := export.ReportPlain(report); err != nil {
return err return err
} }
} }

4
internal/dis/component/exporter/report.go
View file

@ -45,8 +45,8 @@ func (snapshot Snapshot) ReportPlain(w io.Writer) error {
io.WriteString(ww, "======= Errors =======\n") io.WriteString(ww, "======= Errors =======\n")
fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic) fmt.Fprintf(ww, "Panic: %v\n", snapshot.ErrPanic)
fmt.Fprintf(ww, "Start: %s\n", snapshot.ErrStart) fmt.Fprintf(ww, "Start: %v\n", snapshot.ErrStart)
fmt.Fprintf(ww, "Stop: %s\n", snapshot.ErrStop) fmt.Fprintf(ww, "Stop: %v\n", snapshot.ErrStop)
fmt.Fprintf(ww, "Errors: %s\n", snapshot.Errors) fmt.Fprintf(ww, "Errors: %s\n", snapshot.Errors)

2
internal/dis/component/server/templating/assets.go
View file

@ -17,6 +17,6 @@ func (tpl *Templating) CustomAssetPath(name string) string {
func (tpl *Templating) BackupName() string { return "custom" } func (tpl *Templating) BackupName() string { return "custom" }
func (tpl *Templating) Backup(context component.StagingContext) error { func (tpl *Templating) Backup(context *component.StagingContext) error {
return context.CopyDirectory("", tpl.CustomAssetsPath()) return context.CopyDirectory("", tpl.CustomAssetsPath())
} }

2
internal/dis/component/sql/backup.go
View file

@ -16,7 +16,7 @@ func (*SQL) BackupName() string {
} }
// Backup makes a backup of all SQL databases into the path dest. // Backup makes a backup of all SQL databases into the path dest.
func (sql *SQL) Backup(scontext component.StagingContext) error { func (sql *SQL) Backup(scontext *component.StagingContext) error {
return scontext.AddFile("", func(ctx context.Context, file io.Writer) error { return scontext.AddFile("", func(ctx context.Context, file io.Writer) error {
code := sql.Stack().Exec(ctx, stream.NewIOStream(file, scontext.Progress(), nil, 0), "sql", "mysqldump", "--all-databases")() code := sql.Stack().Exec(ctx, stream.NewIOStream(file, scontext.Progress(), nil, 0), "sql", "mysqldump", "--all-databases")()
if code != 0 { if code != 0 {

2
internal/dis/component/sql/snapshot.go
View file

@ -13,7 +13,7 @@ func (*SQL) SnapshotNeedsRunning() bool { return false }
func (*SQL) SnapshotName() string { return "sql" } func (*SQL) SnapshotName() string { return "sql" }
func (sql *SQL) Snapshot(wisski models.Instance, scontext component.StagingContext) error { func (sql *SQL) Snapshot(wisski models.Instance, scontext *component.StagingContext) error {
return scontext.AddDirectory(".", func(ctx context.Context) error { return scontext.AddDirectory(".", func(ctx context.Context) error {
return scontext.AddFile(wisski.SqlDatabase+".sql", func(ctx context.Context, file io.Writer) error { return scontext.AddFile(wisski.SqlDatabase+".sql", func(ctx context.Context, file io.Writer) error {
return sql.SnapshotDB(ctx, scontext.Progress(), file, wisski.SqlDatabase) return sql.SnapshotDB(ctx, scontext.Progress(), file, wisski.SqlDatabase)

2
internal/dis/component/triplestore/backup.go
View file

@ -11,7 +11,7 @@ import (
func (ts *Triplestore) BackupName() string { return "triplestore" } func (ts *Triplestore) BackupName() string { return "triplestore" }
// Backup makes a backup of all Triplestore repositories databases into the path dest. // Backup makes a backup of all Triplestore repositories databases into the path dest.
func (ts *Triplestore) Backup(scontext component.StagingContext) error { func (ts *Triplestore) Backup(scontext *component.StagingContext) error {
return scontext.AddDirectory("", func(ctx context.Context) error { return scontext.AddDirectory("", func(ctx context.Context) error {
// list all the directories // list all the directories
repos, err := ts.listRepositories(ctx) repos, err := ts.listRepositories(ctx)

2
internal/dis/component/triplestore/snapshot.go
View file

@ -14,7 +14,7 @@ func (Triplestore) SnapshotNeedsRunning() bool { return false }
func (Triplestore) SnapshotName() string { return "triplestore" } func (Triplestore) SnapshotName() string { return "triplestore" }
func (ts *Triplestore) Snapshot(wisski models.Instance, scontext component.StagingContext) error { func (ts *Triplestore) Snapshot(wisski models.Instance, scontext *component.StagingContext) error {
return scontext.AddDirectory(".", func(ctx context.Context) error { return scontext.AddDirectory(".", func(ctx context.Context) error {
return scontext.AddFile(wisski.GraphDBRepository+".nq", func(ctx context.Context, file io.Writer) error { return scontext.AddFile(wisski.GraphDBRepository+".nq", func(ctx context.Context, file io.Writer) error {
_, err := ts.SnapshotDB(ctx, file, wisski.GraphDBRepository) _, err := ts.SnapshotDB(ctx, file, wisski.GraphDBRepository)