From 6f2ba182271890a828c605a79cf8cf2866529ecb Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Wed, 21 Sep 2022 15:20:42 +0200 Subject: [PATCH] Fetch Prefixes from Triplestore --- cmd/prefixes.go | 49 +++++++++++++++++ cmd/wdcli/main.go | 1 + internal/component/control/html/instance.html | 7 +++ .../barrel/wisskiutils/list_uri_prefixes.php | 19 ------- .../instances/php/list_uri_prefixes.php | 48 +++++++++++++++++ internal/component/instances/wisski_prefix.go | 53 +++++++++++++++++-- internal/component/instances/wisski_status.go | 5 ++ pkg/slicesx/slicesx.go | 50 +++++++++++++++++ 8 files changed, 209 insertions(+), 23 deletions(-) create mode 100644 cmd/prefixes.go delete mode 100644 internal/component/instances/instances/barrel/wisskiutils/list_uri_prefixes.php create mode 100644 internal/component/instances/php/list_uri_prefixes.php create mode 100644 pkg/slicesx/slicesx.go diff --git a/cmd/prefixes.go b/cmd/prefixes.go new file mode 100644 index 0000000..31b1472 --- /dev/null +++ b/cmd/prefixes.go @@ -0,0 +1,49 @@ +package cmd + +import ( + wisski_distillery "github.com/FAU-CDI/wisski-distillery" + "github.com/FAU-CDI/wisski-distillery/internal/core" + "github.com/tkw1536/goprogram/exit" +) + +// Prefixes is then 'prefixes' command +var Prefixes wisski_distillery.Command = prefixes{} + +type prefixes struct { + Positionals struct { + Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show prefixes for"` + } `positional-args:"true"` +} + +func (prefixes) Description() wisski_distillery.Description { + return wisski_distillery.Description{ + Requirements: core.Requirements{ + NeedsDistillery: true, + }, + Command: "prefixes", + Description: "List all Prefixes for a specific WissKI", + } +} + +var errPrefixesGeneric = exit.Error{ + ExitCode: exit.ExitGeneric, + Message: "Unable to load prefixes", +} + +func (p prefixes) Run(context wisski_distillery.Context) error { + instance, err := context.Environment.Instances().WissKI(p.Positionals.Slug) + if err != nil { + return err + } + + prefixes, err := instance.Prefixes() + if err != nil { + return errPrefixesGeneric.Wrap(err) + } + + for _, p := range prefixes { + context.Println(p) + } + + return nil +} diff --git a/cmd/wdcli/main.go b/cmd/wdcli/main.go index 3033788..a612f3d 100644 --- a/cmd/wdcli/main.go +++ b/cmd/wdcli/main.go @@ -43,6 +43,7 @@ func init() { wdcli.Register(cmd.BlindUpdate) wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration wdcli.Register(cmd.Pathbuilders) + wdcli.Register(cmd.Prefixes) // backup & cron wdcli.Register(cmd.Snapshot) diff --git a/internal/component/control/html/instance.html b/internal/component/control/html/instance.html index 2c2017f..00ff40d 100644 --- a/internal/component/control/html/instance.html +++ b/internal/component/control/html/instance.html @@ -12,6 +12,13 @@ Slug: {{ .Info.Slug }}
URL: {{ .Info.URL }}
+ Resolver Prefixes: + +
Running: {{ .Info.Running }}
diff --git a/internal/component/instances/instances/barrel/wisskiutils/list_uri_prefixes.php b/internal/component/instances/instances/barrel/wisskiutils/list_uri_prefixes.php deleted file mode 100644 index f9a52ba..0000000 --- a/internal/component/instances/instances/barrel/wisskiutils/list_uri_prefixes.php +++ /dev/null @@ -1,19 +0,0 @@ -getStorage('wisski_salz_adapter'); -foreach ($storage->loadMultiple() as $adapter) { - // read the configuration, and check if we have a default graph - $conf = $adapter->getEngine()->getConfiguration(); - if(!array_key_exists('default_graph', $conf)) { - continue; - } - - // and echo it out - echo $conf['default_graph'] . "\n"; -} \ No newline at end of file diff --git a/internal/component/instances/php/list_uri_prefixes.php b/internal/component/instances/php/list_uri_prefixes.php new file mode 100644 index 0000000..6df7248 --- /dev/null +++ b/internal/component/instances/php/list_uri_prefixes.php @@ -0,0 +1,48 @@ +getStorage('wisski_salz_adapter'); + foreach ($storage->loadMultiple() as $adapter) { + // load all the prefixes from the triplestore + $engine = $adapter->getEngine(); + getTriplestorePrefixes($adapter->getEngine(), $prefixes); + + // read the configuration to check if we have a default graph + $conf = $engine->getConfiguration(); + if(!array_key_exists('default_graph', $conf)) { + continue; + } + $prefixes[] = $conf['default_graph']; + } + return $prefixes; +} + +function getTriplestorePrefixes($engine, &$prefixes) { + $results = $engine->directQuery(' + select distinct ?base where { + { + select distinct ?iri where { + { + select distinct (?s as ?iri) { ?s ?p ?o } + } union { + select distinct (?o as ?iri) { ?s ?p ?o FILTER(isiri(?o)) } + } + } + } + BIND(replace(str(?iri), "/[^/]*/?$", "/") as ?base) + FILTER(!REGEX(?base, "/wisski/navigate/[\\\\d]+/$")) + } ORDER BY ?base'); + if (!$results) return FALSE; + + foreach($results as $result) { + $prefixes[] = $result->base->getValue(); + } + + return TRUE; +} + diff --git a/internal/component/instances/wisski_prefix.go b/internal/component/instances/wisski_prefix.go index df255a9..8ff7b6d 100644 --- a/internal/component/instances/wisski_prefix.go +++ b/internal/component/instances/wisski_prefix.go @@ -7,7 +7,10 @@ import ( "strings" "github.com/FAU-CDI/wisski-distillery/pkg/fsx" + "github.com/FAU-CDI/wisski-distillery/pkg/slicesx" "github.com/tkw1536/goprogram/stream" + + _ "embed" ) // NoPrefix checks if this WissKI instance is excluded from generating prefixes. @@ -16,6 +19,43 @@ func (wisski *WissKI) NoPrefix() bool { return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip")) } +//go:embed php/list_uri_prefixes.php +var listURIPrefixesPHP string + +// Prefixes returns the prefixes +func (wisski *WissKI) Prefixes() (prefixes []string, err error) { + // get all the ugly prefixes + err = wisski.ExecPHPScript(stream.FromEnv(), &prefixes, listURIPrefixesPHP, "list_prefixes") + if err != nil { + return nil, err + } + + // filter out sequential prefixes + prefixes = slicesx.NonSequential(prefixes, func(prev, now string) bool { + return strings.HasPrefix(now, prev) + }) + + // filter out blocked prefixes + return slicesx.Filter(prefixes, func(uri string) bool { return !IsNonServedURI(uri) }), nil +} + +// TODO: Eventually move this into a configuration file. +// But for now this is fine +var blockedURIs = []string{ + "http://erlangen-crm.org/", + "http://www.w3.org/", + "xsd:", +} + +func IsNonServedURI(candidate string) bool { + return slicesx.Any( + blockedURIs, + func(prefix string) bool { + return strings.HasPrefix(candidate, prefix) + }, + ) +} + var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") // PrefixConfig returns the prefix config belonging to this instance. @@ -32,10 +72,15 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) { builder.WriteString("\n") // default prefixes - wu := stream.NewIOStream(&builder, nil, nil, 0) - code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php") - if err != nil || code != 0 { - return "", errPrefixExecFailed + prefixes, err := wisski.Prefixes() + if err != nil { + return "", err + } + + // predefined prefixes + for _, prefix := range prefixes { + builder.WriteString(prefix) + builder.WriteRune('\n') } // custom prefixes diff --git a/internal/component/instances/wisski_status.go b/internal/component/instances/wisski_status.go index b8a83b3..d8c8953 100644 --- a/internal/component/instances/wisski_status.go +++ b/internal/component/instances/wisski_status.go @@ -17,6 +17,7 @@ type Info struct { Running bool // is the instance running? Pathbuilders map[string]string // list of pathbuilders + Prefixes []string // list of uri prefixes } // Info returns information about this WissKI instance. @@ -46,6 +47,10 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) { info.LastRebuild, _ = wisski.LastRebuild() return nil }) + group.Go(func() (err error) { + info.Prefixes, _ = wisski.Prefixes() + return nil + }) } err = group.Wait() diff --git a/pkg/slicesx/slicesx.go b/pkg/slicesx/slicesx.go new file mode 100644 index 0000000..37d0ef6 --- /dev/null +++ b/pkg/slicesx/slicesx.go @@ -0,0 +1,50 @@ +package slicesx + +import ( + "golang.org/x/exp/constraints" + "golang.org/x/exp/slices" +) + +// Any returns true if test returns true for any of values. +func Any[T any](values []T, test func(T) bool) bool { + for _, v := range values { + if test(v) { + return true + } + } + return false +} + +// Filter filters values in place +func Filter[T any](values []T, filter func(T) bool) []T { + results := values[:0] + for _, value := range values { + if filter(value) { + results = append(results, value) + } + } + return results +} + +// NonSequential sorts values, and then removes elements for which test() returns true. +// NonSequential does not re-allocate, but uses the existing slice. +func NonSequential[T constraints.Ordered](values []T, test func(prev, current T) bool) []T { + if len(values) < 2 { + return values + } + + // sort the values and make a results array + slices.Sort(values) + results := values[:1] + + // do the filter loop + prev := results[0] + for _, current := range values[1:] { + if !test(prev, current) { + results = append(results, current) + } + prev = current + } + + return results +}