Fetch Prefixes from Triplestore

cmd/prefixes.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	wisski_distillery "github.com/FAU-CDI/wisski-distillery"
+	"github.com/FAU-CDI/wisski-distillery/internal/core"
+	"github.com/tkw1536/goprogram/exit"
+)
+
+// Prefixes is then 'prefixes' command
+var Prefixes wisski_distillery.Command = prefixes{}
+
+type prefixes struct {
+	Positionals struct {
+		Slug string `positional-arg-name:"SLUG" required:"1-1" description:"slug of instance to show prefixes for"`
+	} `positional-args:"true"`
+}
+
+func (prefixes) Description() wisski_distillery.Description {
+	return wisski_distillery.Description{
+		Requirements: core.Requirements{
+			NeedsDistillery: true,
+		},
+		Command:     "prefixes",
+		Description: "List all Prefixes for a specific WissKI",
+	}
+}
+
+var errPrefixesGeneric = exit.Error{
+	ExitCode: exit.ExitGeneric,
+	Message:  "Unable to load prefixes",
+}
+
+func (p prefixes) Run(context wisski_distillery.Context) error {
+	instance, err := context.Environment.Instances().WissKI(p.Positionals.Slug)
+	if err != nil {
+		return err
+	}
+
+	prefixes, err := instance.Prefixes()
+	if err != nil {
+		return errPrefixesGeneric.Wrap(err)
+	}
+
+	for _, p := range prefixes {
+		context.Println(p)
+	}
+
+	return nil
+}

cmd/wdcli/main.go

@@ -43,6 +43,7 @@ func init() {
 	wdcli.Register(cmd.BlindUpdate)
 	wdcli.Register(cmd.UpdatePrefixConfig) // TODO: Move into post-instance configuration
 	wdcli.Register(cmd.Pathbuilders)
+	wdcli.Register(cmd.Prefixes)
 
 	// backup & cron
 	wdcli.Register(cmd.Snapshot)

internal/component/control/html/instance.html

@@ -12,6 +12,13 @@
     <b>Slug:</b> <code>{{ .Info.Slug }}</code> <br />
     <b>URL:</b> <a href="{{ .Info.URL }}" target="_blank" rel="noopener noreferrer">{{ .Info.URL }}</a> <br />
     <hr />
+    <b>Resolver Prefixes:</b>
+    <ul>
+        {{ range .Info.Prefixes }}
+            <li><code>{{ . }}</code></li>
+        {{ end}}
+    </ul>
+    <hr />
     <b>Running:</b> <code>{{ .Info.Running }}</code> <br />
     <!-- <b>OwnerEmail:</b> <code>{{ .Instance.OwnerEmail }}</code> <br /> -->
     <hr />

internal/component/instances/instances/barrel/wisskiutils/list_uri_prefixes.php

@@ -1,19 +0,0 @@
-<?php
-
-/**
- * This script will list all the URIs that this system is aware of.
- * This works by listing all the default graph uris of all the adapters.
- */
-
-// iterate over all adapters
-$storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
-foreach ($storage->loadMultiple() as $adapter) {
-    // read the configuration, and check if we have a default graph
-    $conf = $adapter->getEngine()->getConfiguration();
-    if(!array_key_exists('default_graph', $conf)) {
-        continue;
-    }
-
-    // and echo it out
-    echo $conf['default_graph'] . "\n";
-}

internal/component/instances/php/list_uri_prefixes.php

@@ -0,0 +1,48 @@
+<?php
+
+/**
+ * list_prefixes lists all content prefixes known to this WissKI.
+ * Prefixes are not filtered, and may contain duplicates.
+ */ 
+function list_prefixes() {
+    $prefixes = [];
+    $storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+    foreach ($storage->loadMultiple() as $adapter) {
+        // load all the prefixes from the triplestore
+        $engine = $adapter->getEngine();
+        getTriplestorePrefixes($adapter->getEngine(), $prefixes);
+
+        // read the configuration to check if we have a default graph
+        $conf = $engine->getConfiguration();
+        if(!array_key_exists('default_graph', $conf)) {
+            continue;
+        }
+        $prefixes[] = $conf['default_graph'];
+    }
+    return $prefixes;
+}
+
+function getTriplestorePrefixes($engine, &$prefixes) {
+    $results = $engine->directQuery('
+    select distinct ?base where {
+        {
+            select distinct ?iri where {
+                {
+                    select distinct (?s as ?iri) { ?s ?p ?o  }
+                } union {
+                    select distinct (?o as ?iri) { ?s ?p ?o FILTER(isiri(?o)) }
+                }
+            }  
+        }
+        BIND(replace(str(?iri), "/[^/]*/?$", "/") as ?base)
+        FILTER(!REGEX(?base, "/wisski/navigate/[\\\\d]+/$"))
+    } ORDER BY ?base');
+    if (!$results) return FALSE;
+
+    foreach($results as $result) {
+        $prefixes[] = $result->base->getValue();
+    }
+
+    return TRUE;
+} 
+

internal/component/instances/wisski_prefix.go

@@ -7,7 +7,10 @@ import (
 	"strings"
 
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/FAU-CDI/wisski-distillery/pkg/slicesx"
 	"github.com/tkw1536/goprogram/stream"
+
+	_ "embed"
 )
 
 // NoPrefix checks if this WissKI instance is excluded from generating prefixes.
@@ -16,6 +19,43 @@ func (wisski *WissKI) NoPrefix() bool {
 	return fsx.IsFile(wisski.instances.Environment, filepath.Join(wisski.FilesystemBase, "prefixes.skip"))
 }
 
+//go:embed php/list_uri_prefixes.php
+var listURIPrefixesPHP string
+
+// Prefixes returns the prefixes
+func (wisski *WissKI) Prefixes() (prefixes []string, err error) {
+	// get all the ugly prefixes
+	err = wisski.ExecPHPScript(stream.FromEnv(), &prefixes, listURIPrefixesPHP, "list_prefixes")
+	if err != nil {
+		return nil, err
+	}
+
+	// filter out sequential prefixes
+	prefixes = slicesx.NonSequential(prefixes, func(prev, now string) bool {
+		return strings.HasPrefix(now, prev)
+	})
+
+	// filter out blocked prefixes
+	return slicesx.Filter(prefixes, func(uri string) bool { return !IsNonServedURI(uri) }), nil
+}
+
+// TODO: Eventually move this into a configuration file.
+// But for now this is fine
+var blockedURIs = []string{
+	"http://erlangen-crm.org/",
+	"http://www.w3.org/",
+	"xsd:",
+}
+
+func IsNonServedURI(candidate string) bool {
+	return slicesx.Any(
+		blockedURIs,
+		func(prefix string) bool {
+			return strings.HasPrefix(candidate, prefix)
+		},
+	)
+}
+
 var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes")
 
 // PrefixConfig returns the prefix config belonging to this instance.
@@ -32,10 +72,15 @@ func (wisski *WissKI) PrefixConfig() (config string, err error) {
 	builder.WriteString("\n")
 
 	// default prefixes
-	wu := stream.NewIOStream(&builder, nil, nil, 0)
+	prefixes, err := wisski.Prefixes()
-	code, err := wisski.Barrel().Exec(wu, "barrel", "/bin/bash", "/user_shell.sh", "-c", "drush php:script /wisskiutils/list_uri_prefixes.php")
+	if err != nil {
-	if err != nil || code != 0 {
+		return "", err
-		return "", errPrefixExecFailed
+	}
+
+	// predefined prefixes
+	for _, prefix := range prefixes {
+		builder.WriteString(prefix)
+		builder.WriteRune('\n')
 	}
 
 	// custom prefixes

internal/component/instances/wisski_status.go

@@ -17,6 +17,7 @@ type Info struct {
 
 	Running      bool              // is the instance running?
 	Pathbuilders map[string]string // list of pathbuilders
+	Prefixes     []string          // list of uri prefixes
 }
 
 // Info returns information about this WissKI instance.
@@ -46,6 +47,10 @@ func (wisski *WissKI) Info(quick bool) (info Info, err error) {
 			info.LastRebuild, _ = wisski.LastRebuild()
 			return nil
 		})
+		group.Go(func() (err error) {
+			info.Prefixes, _ = wisski.Prefixes()
+			return nil
+		})
 	}
 
 	err = group.Wait()

pkg/slicesx/slicesx.go

@@ -0,0 +1,50 @@
+package slicesx
+
+import (
+	"golang.org/x/exp/constraints"
+	"golang.org/x/exp/slices"
+)
+
+// Any returns true if test returns true for any of values.
+func Any[T any](values []T, test func(T) bool) bool {
+	for _, v := range values {
+		if test(v) {
+			return true
+		}
+	}
+	return false
+}
+
+// Filter filters values in place
+func Filter[T any](values []T, filter func(T) bool) []T {
+	results := values[:0]
+	for _, value := range values {
+		if filter(value) {
+			results = append(results, value)
+		}
+	}
+	return results
+}
+
+// NonSequential sorts values, and then removes elements for which test() returns true.
+// NonSequential does not re-allocate, but uses the existing slice.
+func NonSequential[T constraints.Ordered](values []T, test func(prev, current T) bool) []T {
+	if len(values) < 2 {
+		return values
+	}
+
+	// sort the values and make a results array
+	slices.Sort(values)
+	results := values[:1]
+
+	// do the filter loop
+	prev := results[0]
+	for _, current := range values[1:] {
+		if !test(prev, current) {
+			results = append(results, current)
+		}
+		prev = current
+	}
+
+	return results
+}