Move WissKI Parts to new ingredients system

internal/wisski/ingredient/php/extras/pathbuilder.go

@@ -0,0 +1,44 @@
+package extras
+
+import (
+	_ "embed"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/php"
+	"golang.org/x/exp/slices"
+)
+
+type Pathbuilder struct {
+	ingredient.Base
+
+	PHP *php.PHP
+}
+
+//go:embed pathbuilder.php
+var pathbuilderPHP string
+
+// All returns the ids of all pathbuilders in consistent order.
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (pathbuilder *Pathbuilder) All(server *php.Server) (ids []string, err error) {
+	err = pathbuilder.PHP.ExecScript(server, &ids, pathbuilderPHP, "all_list")
+	slices.Sort(ids)
+	return
+}
+
+// Get returns a single pathbuilder as xml.
+// If it does not exist, it returns the empty string and nil error.
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (pathbuilder *Pathbuilder) Get(server *php.Server, id string) (xml string, err error) {
+	err = pathbuilder.PHP.ExecScript(server, &xml, pathbuilderPHP, "one_xml", id)
+	return
+}
+
+// GetAll returns all pathbuilders serialized as xml
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (pathbuilder *Pathbuilder) GetAll(server *php.Server) (pathbuilders map[string]string, err error) {
+	err = pathbuilder.PHP.ExecScript(server, &pathbuilders, pathbuilderPHP, "all_xml")
+	return
+}

internal/wisski/ingredient/php/extras/pathbuilder.php

@@ -0,0 +1,77 @@
+<?php
+
+use Drupal\wisski_pathbuilder\Entity\WisskiPathEntity;
+
+/** all_xml lists all pathbuilders, and returns the corresponding xml */
+function all_xml(): object {
+    $all = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->loadMultiple();
+    return (object)array_map("entity_to_xml", $all);
+}
+
+
+/** all_list lists the ids of all pathbuilders */
+function all_list(): Array {
+    return array_keys(\Drupal::entityQuery('wisski_pathbuilder')->execute());
+}
+
+/** one_xml serializes a single pathbuilder as xml */
+function one_xml(string $id): string {
+    $pb = \Drupal::entityTypeManager()->getStorage('wisski_pathbuilder')->load($id);
+    if ($pb === NULL) {
+        return "";
+    }
+    return entity_to_xml($pb);
+}
+
+// =================================================================================
+// =================================================================================
+
+
+function entity_to_xml($pb) {
+    $xml = new \SimpleXMLElement("<pathbuilderinterface></pathbuilderinterface>");
+
+    $paths = $pb->getAllPaths();
+    foreach ($paths as $key => $path) {
+        $id = $path->getID();
+
+        $path = $pb->getPbPath($id);
+
+        $pathChild = $xml->addChild("path");
+        $pathObject = WisskiPathEntity::load($id);
+
+        foreach ($path as $subkey => $value) {
+
+            if (in_array($subkey, ['relativepath'])) {
+                continue;
+            }
+
+            if ($subkey == "parent") {
+                $subkey = "group_id";
+            }
+
+            $pathChild->addChild($subkey, htmlspecialchars($value));
+        }
+
+        $pathArray = $pathChild->addChild('path_array');
+        foreach ($pathObject->getPathArray() as $subkey => $value) {
+            $pathArray->addChild($subkey % 2 == 0 ? 'x' : 'y', $value);
+        }
+
+        $pathChild->addChild('datatype_property', htmlspecialchars($pathObject->getDatatypeProperty()));
+        $pathChild->addChild('short_name', htmlspecialchars($pathObject->getShortName()));
+        $pathChild->addChild('disamb', htmlspecialchars($pathObject->getDisamb()));
+        $pathChild->addChild('description', htmlspecialchars($pathObject->getDescription()));
+        $pathChild->addChild('uuid', htmlspecialchars($pathObject->uuid()));
+        if ($pathObject->getType() == "Group" || $pathObject->getType() == "Smartgroup") {
+            $pathChild->addChild('is_group', "1");
+        } else {
+            $pathChild->addChild('is_group', "0");
+        }
+        $pathChild->addChild('name', htmlspecialchars($pathObject->getName()));
+    }
+
+    // turn it into XML
+    $dom = dom_import_simplexml($xml)->ownerDocument;
+    $dom->formatOutput = TRUE;
+    return $dom->saveXML();
+}

internal/wisski/ingredient/php/extras/prefixes.go

@@ -0,0 +1,155 @@
+package extras
+
+import (
+	"bufio"
+	"path/filepath"
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/mstore"
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/php"
+	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
+	"github.com/tkw1536/goprogram/lib/collection"
+
+	_ "embed"
+)
+
+// Prefixes implements reading and writing prefix
+type Prefixes struct {
+	ingredient.Base
+
+	PHP    *php.PHP
+	MStore *mstore.MStore
+}
+
+// NoPrefix checks if this WissKI instance is excluded from generating prefixes.
+// TODO: Move this to the database!
+func (prefixes *Prefixes) NoPrefix() bool {
+	return fsx.IsFile(prefixes.Malt.Environment, filepath.Join(prefixes.FilesystemBase, "prefixes.skip"))
+}
+
+//go:embed prefixes.php
+var listURIPrefixesPHP string
+
+// All returns the prefixes applying to this WissKI
+//
+// server is an optional server to fetch prefixes from.
+// server may be nil.
+func (prefixes *Prefixes) All(server *php.Server) ([]string, error) {
+	uris, err := prefixes.database(server)
+	if err != nil {
+		return nil, err
+	}
+
+	uris2, err := prefixes.filePrefixes()
+	if err != nil {
+		return nil, err
+	}
+
+	return append(uris, uris2...), nil
+}
+
+func (wisski *Prefixes) database(server *php.Server) (prefixes []string, err error) {
+	// get all the ugly prefixes
+	err = wisski.PHP.ExecScript(server, &prefixes, listURIPrefixesPHP, "list_prefixes")
+	if err != nil {
+		return nil, err
+	}
+
+	// filter out sequential prefixes
+	prefixes = collection.NonSequential(prefixes, func(prev, now string) bool {
+		return strings.HasPrefix(now, prev)
+	})
+
+	// load the list of blocked prefixes
+	blocks, err := wisski.blocked()
+	if err != nil {
+		return nil, err
+	}
+
+	// filter out blocked prefixes
+	return collection.Filter(prefixes, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil
+}
+
+func (prefixes *Prefixes) blocked() ([]string, error) {
+	// open the resolver block file
+	// TODO: move this to the distillery
+	file, err := prefixes.Malt.Environment.Open(prefixes.Malt.Config.SelfResolverBlockFile)
+	if err != nil {
+		return nil, err
+	}
+
+	var lines []string
+
+	// read all the lines that aren't a comment!
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "#") {
+			continue
+		}
+		lines = append(lines, line)
+	}
+
+	// check if there was an error
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	// and done!
+	return lines, nil
+}
+
+func hasAnyPrefix(candidate string, prefixes []string) bool {
+	return collection.Any(
+		prefixes,
+		func(prefix string) bool {
+			return strings.HasPrefix(candidate, prefix)
+		},
+	)
+}
+
+func (wisski *Prefixes) filePrefixes() (prefixes []string, err error) {
+	path := filepath.Join(wisski.FilesystemBase, "prefixes")
+	if !fsx.IsFile(wisski.Malt.Environment, path) {
+		return nil, nil
+	}
+
+	file, err := wisski.Malt.Environment.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		prefixes = append(prefixes, line)
+	}
+
+	if scanner.Err() != nil {
+		return nil, scanner.Err()
+	}
+	return prefixes, nil
+}
+
+// CACHING
+
+var prefix = mstore.For[string]("prefix")
+
+// Prefixes returns the cached prefixes from the given instance
+func (wisski *Prefixes) PrefixesCached() (results []string, err error) {
+	return prefix.GetAll(wisski.MStore)
+}
+
+// UpdatePrefixes updates the cached prefixes of this instance
+func (wisski *Prefixes) UpdatePrefixes() error {
+	prefixes, err := wisski.All(nil)
+	if err != nil {
+		return err
+	}
+	return prefix.SetAll(wisski.MStore, prefixes...)
+}

internal/wisski/ingredient/php/extras/prefixes.php

@@ -0,0 +1,51 @@
+<?php
+
+/**
+ * list_prefixes lists all content prefixes known to this WissKI.
+ * Prefixes are not filtered, and may contain duplicates.
+ */ 
+function list_prefixes() {
+    $prefixes = [];
+    $storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter');
+    foreach ($storage->loadMultiple() as $adapter) {
+        // load all the prefixes from the triplestore
+        $engine = $adapter->getEngine();
+        getTriplestorePrefixes($adapter->getEngine(), $prefixes);
+
+        // read the configuration to check if we have a default graph
+        $conf = $engine->getConfiguration();
+        if(!array_key_exists('default_graph', $conf)) {
+            continue;
+        }
+        $prefixes[] = $conf['default_graph'];
+    }
+    return $prefixes;
+}
+
+function getTriplestorePrefixes($engine, &$prefixes) {
+    // some adapters don't support a query method!
+    if (!method_exists($engine, 'directQuery')) return NULL;
+
+    $results = $engine->directQuery('
+    select distinct ?base where {
+        {
+            select distinct ?iri where {
+                {
+                    select distinct (?s as ?iri) { ?s ?p ?o  }
+                } union {
+                    select distinct (?o as ?iri) { ?s ?p ?o FILTER(isiri(?o)) }
+                }
+            }  
+        }
+        BIND(replace(str(?iri), "/[^/]*/?$", "/") as ?base)
+        FILTER(!REGEX(?base, "/wisski/navigate/[\\\\d]+/$"))
+    } ORDER BY ?base');
+    if (!$results) return FALSE;
+
+    foreach($results as $result) {
+        $prefixes[] = $result->base->getValue();
+    }
+
+    return TRUE;
+} 
+

internal/wisski/ingredient/php/extras/settings.go

@@ -0,0 +1,26 @@
+package extras
+
+import (
+	_ "embed"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/php"
+)
+
+type Settings struct {
+	ingredient.Base
+
+	PHP *php.PHP
+}
+
+//go:embed settings.php
+var settingsPHP string
+
+func (settings *Settings) Get(server *php.Server, key string) (value any, err error) {
+	err = settings.PHP.ExecScript(server, &value, settingsPHP, "get_setting", key)
+	return
+}
+
+func (settings *Settings) Set(server *php.Server, key string, value any) error {
+	return settings.PHP.ExecScript(server, nil, settingsPHP, "set_setting", key, value)
+}

internal/wisski/ingredient/php/extras/settings.php

@@ -0,0 +1,29 @@
+<?php
+
+/** gets a setting from 'settings.php' */
+function get_setting($name) {
+    use \Drupal\Core\Site\Settings;
+    return Settings::get($name);
+}
+
+/** sets a setting in 'settings.php' */
+function set_setting($name, $value) {
+    // load install.inc
+    if(is_file(DRUPAL_ROOT . "/internal/")) {
+        include_once DRUPAL_ROOT . "/internal/core/includes/install.inc";
+    } else {
+        include_once DRUPAL_ROOT . "/core/includes/install.inc";
+    }
+    
+    // update the provided setting
+    $settings["settings"][$name] = (object)[
+        "value" => $value,
+        "required" => TRUE,
+    ];
+
+    // find the filename
+    $filename = DRUPAL_ROOT . "/" . \Drupal::service("site.path") . "/settings.php";
+    drupal_rewrite_settings($settings, $filename);
+
+    return True;
+}

internal/wisski/ingredient/php/php.go

@@ -0,0 +1,58 @@
+package php
+
+import (
+	"strings"
+
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient"
+	"github.com/FAU-CDI/wisski-distillery/internal/wisski/ingredient/barrel"
+)
+
+type PHP struct {
+	ingredient.Base
+
+	Barrel *barrel.Barrel
+}
+
+// ExecScript executes the PHP code as a script on the given server.
+// When server is nil, creates a new server and automatically closes it after execution.
+// Calling this function repeatedly with server = nil is inefficient.
+//
+// The script should define a function called entrypoint, and may define additional functions.
+//
+// Code must start with "<?php" and may not contain a closing tag.
+// Code is expected not to mess with PHPs output buffer.
+// Code should not contain user input.
+// Code breaking these conventions may or may not result in an error.
+//
+// It's arguments are encoded as json using [json.Marshal] and decoded within php.
+//
+// The return value of the function is again marshaled with json and returned to the caller.
+func (php *PHP) ExecScript(server *Server, value any, code string, entrypoint string, args ...any) (err error) {
+	if server == nil {
+		server, err = php.NewServer()
+		if err != nil {
+			return
+		}
+		defer server.Close()
+	}
+
+	if code != "" {
+		if err := server.MarshalEval(nil, strings.TrimPrefix(code, "<?php")); err != nil {
+			return err
+		}
+	}
+
+	return server.MarshalCall(value, entrypoint, args...)
+}
+
+func (php *PHP) EvalCode(server *Server, value any, code string) (err error) {
+	if server == nil {
+		server, err = php.NewServer()
+		if err != nil {
+			return
+		}
+		defer server.Close()
+	}
+
+	return server.MarshalEval(value, code)
+}

internal/wisski/ingredient/php/server.go

@@ -0,0 +1,268 @@
+package php
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	_ "embed"
+
+	"github.com/alessio/shellescape"
+	"github.com/tkw1536/goprogram/lib/collection"
+	"github.com/tkw1536/goprogram/lib/nobufio"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Common PHP Error
+var (
+	errPHPInit    = "Unable to initialize"
+	errPHPMarshal = "Marshal failed"
+	errPHPInvalid = ServerError{Message: "Invalid code to execute"}
+	errPHPReceive = "Failed to receive response"
+	errPHPClosed  = ServerError{Message: "Server closed"}
+)
+
+// PHPError represents an error during PHPServer logic
+type ServerError struct {
+	Message string
+	Err     error
+}
+
+func (err ServerError) Unwrap() error {
+	return err.Err
+}
+
+func (err ServerError) Error() string {
+	if err.Err == nil {
+		return fmt.Sprintf("PHPServer: %s", err.Message)
+	}
+	return fmt.Sprintf("PHPServer: %s: %s", err.Message, err.Err)
+}
+
+// Throwable represents an error during php code
+type Throwable string
+
+func (throwable Throwable) Error() string {
+	return string(throwable)
+}
+
+// NewServer returns a new server that can execute code within this distillery.
+// When err == nil, the caller must call server.Close().
+//
+// See [PHPServer].
+func (php *PHP) NewServer() (*Server, error) {
+	// create input and output pipes
+	ir, iw, err := os.Pipe()
+	if err != nil {
+		return nil, ServerError{errPHPInit, err}
+	}
+	or, ow, err := os.Pipe()
+	if err != nil {
+		ir.Close()
+		iw.Close()
+		return nil, ServerError{errPHPInit, err}
+	}
+
+	// create a context to close the server
+	context, cancel := context.WithCancel(context.Background())
+
+	// start the shell process, which will close everything once done
+	go func() {
+		defer func() {
+			ir.Close()
+			iw.Close()
+			or.Close()
+			ow.Close()
+
+			cancel()
+		}()
+
+		// start the server
+		io := stream.NewIOStream(ow, nil, ir, 0)
+		php.Barrel.Shell(io, "-c", shellescape.QuoteCommand([]string{"drush", "php:eval", serverPHP}))
+	}()
+
+	// return the seerver
+	return &Server{
+		in:  iw,
+		out: or,
+		c:   context,
+	}, nil
+}
+
+// Server represents a server that executes code within a distillery.
+// A typical use-case is to define functions using [MarshalEval], and then call those functions [MarshalCall].
+//
+// A nil Server will return [ErrServerBroken] on every function call.
+type Server struct {
+	m sync.Mutex
+
+	in  io.WriteCloser
+	out io.Reader
+	c   context.Context
+}
+
+// MarshalEval evaluates code on the server and Marshals the result into value.
+// When value is nil, the results are discarded.
+//
+// code is directly passed to php's "eval" function.
+// as such any functions defined will remain in server memory.
+//
+// When an exception is thrown by the PHP Code, error is not nil, and dest remains unchanged.
+func (server *Server) MarshalEval(value any, code string) error {
+	server.m.Lock()
+	defer server.m.Unlock()
+
+	// quick hack: when the server is already done
+	if err := server.c.Err(); err != nil {
+		return errPHPClosed
+	}
+
+	// marshal the code, and send it to the server
+	bytes, err := json.Marshal(code)
+	if err != nil {
+		return ServerError{Message: errPHPMarshal, Err: err}
+	}
+
+	// send it to the server
+	io.WriteString(server.in, string(bytes)+"\n")
+
+	// read the next line (as a response)
+	data, err := nobufio.ReadLine(server.out)
+	if err != nil {
+		return ServerError{Message: errPHPReceive, Err: err}
+	}
+
+	// read whatever we received
+	var received [2]json.RawMessage
+	if err := json.Unmarshal([]byte(data), &received); err != nil {
+		return ServerError{Message: errPHPMarshal, Err: err}
+	}
+
+	// check if there was an error
+	var errString string
+	if err := json.Unmarshal(received[1], &errString); err == nil && errString != "" {
+		return Throwable(errString)
+	}
+
+	// special case: no return value => no unmarshaling needed
+	if value == nil {
+		return nil
+	}
+
+	// read the actual result!
+	return json.Unmarshal(received[0], value)
+}
+
+// Eval is like [MarshalEval], but returns the value as an any
+func (server *Server) Eval(code string) (value any, err error) {
+	err = server.MarshalEval(&value, code)
+	return
+}
+
+// MarshalCall calls a previously defined function with the given arguments.
+// Arguments are sent to php using json Marshal, and are 'json_decode'd on the php side.
+//
+// Return values are received as in [MarshalEval].
+func (server *Server) MarshalCall(value any, function string, args ...any) error {
+	// marshal a code for the call
+	userFunction, err := marshalPHP(function)
+	if err != nil {
+		return ServerError{Message: errPHPMarshal, Err: err}
+	}
+
+	userFunctionArgs := "[]"
+	if len(args) > 0 {
+		userFunctionArgs, err = marshalPHP(args)
+		if err != nil {
+			return ServerError{Message: errPHPMarshal, Err: err}
+		}
+	}
+	code := "return call_user_func_array(" + userFunction + "," + userFunctionArgs + ");"
+
+	// and return the evaluated code!
+	return server.MarshalEval(value, code)
+}
+
+// Call is like [MarshalCall] but returns the return value of the function as an any
+func (server *Server) Call(function string, args ...any) (value any, err error) {
+	err = server.MarshalCall(&value, function, args...)
+	return
+}
+
+const marshalRune = 'F' // press to pay respect
+
+// marshalPHP marshals some data which can be marshaled using [json.Encode] into a PHP Expression.
+// the string can be safely used directly within php.
+func marshalPHP(data any) (string, error) {
+	// this function uses json as a data format to transport the data into php.
+	// then we build a heredoc to encode it safely, and decode it in php
+
+	// Step 1: Encode the data as json
+	jbytes, err := json.Marshal(data)
+	if err != nil {
+		return "", err
+	}
+	jstring := string(jbytes)
+
+	// Step 2: Find a delimiter for the heredoc.
+	// Step 2a: Find the longest sequence of [marshalRune]s inside the encoded string.
+	var current, longest int
+	for _, r := range jstring {
+
+		if r == marshalRune {
+			current++
+		} else {
+			current = 0
+		}
+
+		if current > longest {
+			longest = current
+		}
+	}
+	// Step 2b: Build a string of marshalRune that is one longer!
+	delim := strings.Repeat(string(marshalRune), longest+1)
+
+	// Step 3: Assemble the encoded string!
+	result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
+	return result, nil
+}
+
+// Close closes this server and prevents any further code from being run.
+func (server *Server) Close() error {
+	server.m.Lock()
+	defer server.m.Unlock()
+
+	// if the context is already closed
+	if err := server.c.Err(); err != nil {
+		return errPHPClosed
+	}
+
+	server.in.Close()
+	<-server.c.Done()
+
+	return nil
+}
+
+//go:embed server.php
+var serverPHP string
+
+// pre-process the server.php code to make it shorter
+func init() {
+	// remove the first '<?php' line
+	lines := strings.Split(serverPHP, "\n")[1:]
+	for i, line := range lines {
+		lines[i] = strings.TrimSpace(line)
+	}
+
+	// remove comment lines
+	lines = collection.Filter(lines, func(line string) bool {
+		return !strings.HasPrefix(line, "//")
+	})
+
+	serverPHP = strings.Join(lines, "")
+}

internal/wisski/ingredient/php/server.php

@@ -0,0 +1,34 @@
+<?php
+
+// This file contains code to execute a php execution server.
+// It is passed as a *command line literal * directly to 'drush:script'.
+//
+// As such it is preprocessed and shortened.
+// It should only contain comments at the beginning of each line, and only starting with '//'.
+// See wisski_php_server.go.
+
+// don't buffer stdin!
+stream_set_read_buffer(STDIN,0);
+
+// stop all other output
+ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
+
+while($line =  fgets(STDIN)){
+    // decode the command to run
+	$code=@json_decode($line);
+
+    // execute it
+	try{
+		$json = json_encode([eval($code),""]);
+	}catch(Throwable $t){
+		$json = json_encode([null,(string)$t]);
+	}
+	if($json===false) {
+		$json = '[null,"Error encoding result"]';
+	}
+
+    // and write out the result
+	ob_end_clean();
+	fwrite(STDOUT,"$json\n");
+    ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
+}