From c4de1f2a06629c3045920362859644ee348f5cfc Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Fri, 9 Sep 2022 10:12:18 +0200 Subject: [PATCH] snapshot: Explicitly export Pathbuilders This commit updates the 'wdcli snapshot' command to also export pathbuilders from the system. --- README.md | 9 +-- TODO.md | 2 - .../barrel/wisskiutils/export_pathbuilder.php | 63 +++++++++++++++++++ env/instances.go | 38 +++++++++++ env/snapshot.go | 19 ++++++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 distillery/resources/compose/barrel/wisskiutils/export_pathbuilder.php diff --git a/README.md b/README.md index 5e7cf15..352cda1 100644 --- a/README.md +++ b/README.md @@ -288,15 +288,16 @@ sudo /var/www/deploy/wdcli snapshot SLUG sudo /var/www/deploy/wdcli snapshot SLUG /path/to/snapshot.tar.gz ``` -The backup proceeeds as follows: +The snapshot proceeeds as follows: 1. make a copy of the instance configuration 2. shutdown the running instance 3. make a dump of the triplestore and mysql databases 4. make a copy of the file system -5. start the instance again -6. package the data into the final `.tar.gz` file +5. export all pathbuilders +6. start the instance again +7. package the data into the final `.tar.gz` file -When uptime is critical, it is possible to skip sets 2 and 5 and leave the instance running. +When uptime is critical, it is possible to skip shutting down a running instance. This might result in inconsistent backup data. To do so, run the script with the `--keepalive` flag: diff --git a/TODO.md b/TODO.md index 0d1d306..3eb625c 100644 --- a/TODO.md +++ b/TODO.md @@ -9,8 +9,6 @@ - Move `provision_entrypoint.sh` into go - Enhance Snapshots - Export the Docker Images - - Export the XML from the Pathbuilder - - Snapshot the docker images being used also! - Avoid running `docker compose` executable and shift it to a library - Cleanup code: Have consistent error handling - Add a metadata / statistics server diff --git a/distillery/resources/compose/barrel/wisskiutils/export_pathbuilder.php b/distillery/resources/compose/barrel/wisskiutils/export_pathbuilder.php new file mode 100644 index 0000000..983e655 --- /dev/null +++ b/distillery/resources/compose/barrel/wisskiutils/export_pathbuilder.php @@ -0,0 +1,63 @@ +getStorage('wisski_pathbuilder')->loadMultiple(); + +// map over the pathbuilders +$xmls = array_map(function($pb) { + $xml = new \SimpleXMLElement(""); + + $paths = $pb->getAllPaths(); + foreach ($paths as $key => $path) { + $id = $path->getID(); + + $path = $pb->getPbPath($id); + + $pathChild = $xml->addChild("path"); + $pathObject = WisskiPathEntity::load($id); + + foreach ($path as $subkey => $value) { + + if (in_array($subkey, ['relativepath'])) { + continue; + } + + if ($subkey == "parent") { + $subkey = "group_id"; + } + + $pathChild->addChild($subkey, htmlspecialchars($value)); + } + + $pathArray = $pathChild->addChild('path_array'); + foreach ($pathObject->getPathArray() as $subkey => $value) { + $pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value); + } + + $pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty())); + $pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName())); + $pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb())); + $pathChild->addChild('description', htmlspecialchars($pathObject->getDescription())); + $pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid())); + if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") { + $pathChild->addChild('is_group', "1"); + } else { + $pathChild->addChild('is_group', "0"); + } + $pathChild->addChild('name', htmlspecialchars($pathObject->getName())); + } + + // turn it into XML + $dom = dom_import_simplexml($xml)->ownerDocument; + $dom->formatOutput = TRUE; + return $dom->saveXML(); +}, $pbs); + +echo json_encode($xmls); \ No newline at end of file diff --git a/env/instances.go b/env/instances.go index ea94689..d9aee0e 100644 --- a/env/instances.go +++ b/env/instances.go @@ -1,6 +1,8 @@ package env import ( + "bytes" + "encoding/json" "fmt" "io" "io/fs" @@ -16,6 +18,8 @@ import ( "github.com/pkg/errors" "github.com/tkw1536/goprogram/exit" "github.com/tkw1536/goprogram/stream" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -366,3 +370,37 @@ func (instance *Instance) PrefixConfig() (config string, err error) { // and done! return builder.String(), nil } + +var errPathbuildersExecFailed = errors.New("ExportPathbuilders: Failed to call export_pathbuilder") + +// ExportPathbuilders writes pathbuilders into the directory dest +func (instance *Instance) ExportPathbuilders(dest string) error { + // export all the pathbuilders into the buffer + var buffer bytes.Buffer + wu := stream.NewIOStream(&buffer, nil, nil, 0) + code, err := instance.Stack().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/export_pathbuilder.php") + if err != nil || code != 0 { + return errPathbuildersExecFailed + } + + // decode them as a json array + var pathbuilders map[string]string + if err := json.NewDecoder(&buffer).Decode(&pathbuilders); err != nil { + return err + } + + // sort the names of the pathbuilders + names := maps.Keys(pathbuilders) + slices.Sort(names) + + // write each into a file! + for _, name := range names { + pbxml := []byte(pathbuilders[name]) + name := filepath.Join(dest, fmt.Sprintf("%s.xml", name)) + if err := os.WriteFile(name, pbxml, fs.ModePerm); err != nil { + return err + } + } + + return nil +} diff --git a/env/snapshot.go b/env/snapshot.go index 18c4781..9db1e07 100644 --- a/env/snapshot.go +++ b/env/snapshot.go @@ -86,6 +86,7 @@ type Snapshot struct { ErrStop error ErrBookkeep error + ErrPathbuilder error ErrFilesystem error ErrTriplestore error ErrSSQL error @@ -145,6 +146,24 @@ func (snapshot *Snapshot) create(io stream.IOStream, instance Instance) { _, snapshot.ErrBookkeep = fmt.Fprintf(info, "%#v\n", instance.Instance) }() + // write pathbuilders + wg.Add(1) + go func() { + defer wg.Done() + + pbPath := filepath.Join(snapshot.Description.Dest, "pathbuilders") + messages <- pbPath + + // create the directory! + if err := os.Mkdir(pbPath, fs.ModeDir); err != nil { + snapshot.ErrPathbuilder = err + return + } + + // put in all the pathbuilders + snapshot.ErrPathbuilder = instance.ExportPathbuilders(pbPath) + }() + // backup the filesystem wg.Add(1) go func() {