rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rebuild: Add on-the-fly compression

Browse source
This commit is contained in:
Tom Wiesing 2024-04-04 14:42:55 +02:00
parent 674b9d8d07
commit e5100a1f22
No known key found for this signature in database
2 changed files with 38 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/rebuild_ts.go
View file

@ -31,5 +31,5 @@ func (rts rebuildTS) Run(context wisski_distillery.Context) (err error) {
return err
}
return instance.TRB().DoSomething(context.Context, context.Stdout, rts.AllowEmptyRepository)
return instance.TRB().RebuildTriplestore(context.Context, context.Stdout, rts.AllowEmptyRepository)
}

51
internal/wisski/ingredient/trb/trb.go
View file

@ -1,6 +1,7 @@
package trb
import (
"compress/gzip"
"context"
"errors"
"fmt"
@ -20,7 +21,8 @@ type TRB struct {
}
}
func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyRepository bool) (err error) {
// RebuildTriplestore rebuilds the triplestore by making a backup, storing it on disk, purging the triplestore, and restoring the backup.
func (trb *TRB) RebuildTriplestore(ctx context.Context, out io.Writer, allowEmptyRepository bool) (err error) {
// stop instance, restart when done
logging.LogMessage(out, "Shutting down instance")
@ -37,12 +39,12 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
}()
// make the backup
logging.LogMessage(out, "Dumping triplestore")
path, err := trb.makeBackup(ctx, allowEmptyRepository)
logging.LogMessage(out, "Storing triplestore content")
dumpPath, err := trb.makeBackup(ctx, allowEmptyRepository)
if err != nil {
return err
}
fmt.Printf("Wrote %q\n", path)
fmt.Printf("Wrote %q\n", dumpPath)
logging.LogMessage(out, "Purging triplestore")
if err := trb.Malt.TS.Purge(ctx, trb.Instance, trb.Domain()); err != nil {
@ -54,15 +56,13 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
return err
}
logging.LogMessage(out, "Loading dump file")
content, err := os.Open(path)
if err != nil {
logging.LogMessage(out, "Restoring triplestore")
if err := trb.restoreBackup(ctx, dumpPath); err != nil {
return err
}
defer content.Close()
logging.LogMessage(out, "Restoring triplestore")
if err := trb.Malt.TS.RestoreDB(ctx, trb.GraphDBRepository, content); err != nil {
logging.LogMessage(out, "Deleting dump file")
if err := os.Remove(dumpPath); err != nil {
return err
}
@ -72,13 +72,17 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
var errBackupEmpty = errors.New("no data contained in backup file (is the repository empty?)")
func (trb *TRB) makeBackup(ctx context.Context, allowEmptyRepository bool) (path string, err error) {
f, err := os.CreateTemp("", "")
file, err := os.CreateTemp("", "*.nq.gz")
if err != nil {
return "", err
}
defer f.Close()
defer file.Close()
count, err := trb.Malt.TS.SnapshotDB(ctx, f, trb.GraphDBRepository)
// create a new writer
zippedFile := gzip.NewWriter(file)
defer zippedFile.Close()
count, err := trb.Malt.TS.SnapshotDB(ctx, zippedFile, trb.GraphDBRepository)
if err != nil {
return "", err
}
@ -87,5 +91,24 @@ func (trb *TRB) makeBackup(ctx context.Context, allowEmptyRepository bool) (path
return "", errBackupEmpty
}
return f.Name(), nil
return file.Name(), nil
}
func (trb *TRB) restoreBackup(ctx context.Context, path string) (err error) {
reader, err := os.Open(path)
if err != nil {
return err
}
defer reader.Close()
decompressedReader, err := gzip.NewReader(reader)
if err != nil {
return err
}
defer decompressedReader.Close()
if err := trb.Malt.TS.RestoreDB(ctx, trb.GraphDBRepository, decompressedReader); err != nil {
return err
}
return nil
}