rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rebuild: Add on-the-fly compression

Browse source
This commit is contained in:
Tom Wiesing 2024-04-04 14:42:55 +02:00
parent 674b9d8d07
commit e5100a1f22
No known key found for this signature in database
2 changed files with 38 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/rebuild_ts.go
View file

@ -31,5 +31,5 @@ func (rts rebuildTS) Run(context wisski_distillery.Context) (err error) {
return err return err
} }
return instance.TRB().DoSomething(context.Context, context.Stdout, rts.AllowEmptyRepository) return instance.TRB().RebuildTriplestore(context.Context, context.Stdout, rts.AllowEmptyRepository)
} }

51
internal/wisski/ingredient/trb/trb.go
View file

@ -1,6 +1,7 @@
package trb package trb
import ( import (
"compress/gzip"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@ -20,7 +21,8 @@ type TRB struct {
} }
} }
func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyRepository bool) (err error) { // RebuildTriplestore rebuilds the triplestore by making a backup, storing it on disk, purging the triplestore, and restoring the backup.
func (trb *TRB) RebuildTriplestore(ctx context.Context, out io.Writer, allowEmptyRepository bool) (err error) {
// stop instance, restart when done // stop instance, restart when done
logging.LogMessage(out, "Shutting down instance") logging.LogMessage(out, "Shutting down instance")
@ -37,12 +39,12 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
}() }()
// make the backup // make the backup
logging.LogMessage(out, "Dumping triplestore") logging.LogMessage(out, "Storing triplestore content")
path, err := trb.makeBackup(ctx, allowEmptyRepository) dumpPath, err := trb.makeBackup(ctx, allowEmptyRepository)
if err != nil { if err != nil {
return err return err
} }
fmt.Printf("Wrote %q\n", path) fmt.Printf("Wrote %q\n", dumpPath)
logging.LogMessage(out, "Purging triplestore") logging.LogMessage(out, "Purging triplestore")
if err := trb.Malt.TS.Purge(ctx, trb.Instance, trb.Domain()); err != nil { if err := trb.Malt.TS.Purge(ctx, trb.Instance, trb.Domain()); err != nil {
@ -54,15 +56,13 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
return err return err
} }
logging.LogMessage(out, "Loading dump file") logging.LogMessage(out, "Restoring triplestore")
content, err := os.Open(path) if err := trb.restoreBackup(ctx, dumpPath); err != nil {
if err != nil {
return err return err
} }
defer content.Close()
logging.LogMessage(out, "Restoring triplestore") logging.LogMessage(out, "Deleting dump file")
if err := trb.Malt.TS.RestoreDB(ctx, trb.GraphDBRepository, content); err != nil { if err := os.Remove(dumpPath); err != nil {
return err return err
} }
@ -72,13 +72,17 @@ func (trb *TRB) DoSomething(ctx context.Context, out io.Writer, allowEmptyReposi
var errBackupEmpty = errors.New("no data contained in backup file (is the repository empty?)") var errBackupEmpty = errors.New("no data contained in backup file (is the repository empty?)")
func (trb *TRB) makeBackup(ctx context.Context, allowEmptyRepository bool) (path string, err error) { func (trb *TRB) makeBackup(ctx context.Context, allowEmptyRepository bool) (path string, err error) {
f, err := os.CreateTemp("", "") file, err := os.CreateTemp("", "*.nq.gz")
if err != nil { if err != nil {
return "", err return "", err
} }
defer f.Close() defer file.Close()
count, err := trb.Malt.TS.SnapshotDB(ctx, f, trb.GraphDBRepository) // create a new writer
zippedFile := gzip.NewWriter(file)
defer zippedFile.Close()
count, err := trb.Malt.TS.SnapshotDB(ctx, zippedFile, trb.GraphDBRepository)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -87,5 +91,24 @@ func (trb *TRB) makeBackup(ctx context.Context, allowEmptyRepository bool) (path
return "", errBackupEmpty return "", errBackupEmpty
} }
return f.Name(), nil return file.Name(), nil
}
func (trb *TRB) restoreBackup(ctx context.Context, path string) (err error) {
reader, err := os.Open(path)
if err != nil {
return err
}
defer reader.Close()
decompressedReader, err := gzip.NewReader(reader)
if err != nil {
return err
}
defer decompressedReader.Close()
if err := trb.Malt.TS.RestoreDB(ctx, trb.GraphDBRepository, decompressedReader); err != nil {
return err
}
return nil
} }