add wisski_php_server

internal/component/instances/php/server.php

@@ -0,0 +1,34 @@
+<?php
+
+// This file contains code to execute a php execution server.
+// It is passed as a *command line literal * directly to 'drush:script'.
+//
+// As such it is preprocessed and shortened.
+// It should only contain comments at the beginning of each line, and only starting with '//'.
+// See wisski_php_server.go.
+
+// don't buffer stdin!
+stream_set_read_buffer(STDIN,0);
+
+// stop all other output
+ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
+
+while($line =  fgets(STDIN)){
+    // decode the command to run
+	$code=@json_decode($line);
+
+    // execute it
+	try{
+		$json = json_encode([eval($code),""]);
+	}catch(Throwable $t){
+		$json = json_encode([null,(string)$t]);
+	}
+	if($json===false) {
+		$json = '[null,"Error encoding result"]';
+	}
+
+    // and write out the result
+	ob_end_clean();
+	fwrite(STDOUT,"$json\n");
+    ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
+}

internal/component/instances/wisski_info.go

@@ -34,57 +34,72 @@ type WissKIInfo struct {
 
 // Info generate a
 func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
-	// TODO: Cache this, and run it with every cron!
+	var group errgroup.Group
+	wisski.infoQuick(&info, &group)
 
+	if !quick {
+		server, err := wisski.NewPHPServer()
+		if err == nil {
+			wisski.infoSlow(&info, server, &group)
+			defer server.Close()
+		}
+	}
+
+	err = group.Wait()
+	return
+}
+
+func (wisski *WissKI) infoQuick(info *WissKIInfo, group *errgroup.Group) {
 	info.Time = time.Now().UTC()
-
-	// static properties
 	info.Slug = wisski.Slug
 	info.URL = wisski.URL().String()
 
-	// dynamic properties
-	var group errgroup.Group
-
-	// quick check if this wisski is running
 	group.Go(func() (err error) {
 		info.Running, err = wisski.Running()
 		return
 	})
 
-	// quick check if this instance is locked
 	group.Go(func() (err error) {
 		info.Locked = wisski.IsLocked()
+		return
+	})
+
+	group.Go(func() (err error) {
+		info.LastRebuild, _ = wisski.LastRebuild()
+		return
+	})
+
+	group.Go(func() (err error) {
+		info.LastUpdate, _ = wisski.LastUpdate()
+		return
+	})
+
+	group.Go(func() (err error) {
+		info.LastRebuild, _ = wisski.LastRebuild()
+		return
+	})
+
+	group.Go(func() (err error) {
+		info.NoPrefixes = wisski.NoPrefix()
+		return
+	})
+}
+
+func (wisski *WissKI) infoSlow(info *WissKIInfo, server *PHPServer, group *errgroup.Group) {
+	group.Go(func() (err error) {
+		info.Prefixes, _ = wisski.Prefixes(server)
 		return nil
 	})
 
-	// slower checks for extra properties.
-	// these might execute php code or require additional database queries.
-	if !quick {
-		group.Go(func() (err error) {
-			info.LastRebuild, _ = wisski.LastRebuild()
-			return nil
-		})
-		group.Go(func() (err error) {
-			info.LastUpdate, _ = wisski.LastUpdate()
-			return nil
-		})
-		group.Go(func() error {
-			info.Pathbuilders, _ = wisski.AllPathbuilders()
-			return nil
-		})
-		group.Go(func() (err error) {
-			info.Prefixes, _ = wisski.Prefixes()
-			info.NoPrefixes = wisski.NoPrefix()
-			return nil
-		})
-		group.Go(func() error {
-			info.Snapshots, _ = wisski.Snapshots()
-			return nil
-		})
-	}
+	group.Go(func() error {
+		info.Snapshots, _ = wisski.Snapshots()
+		return nil
+	})
 
-	err = group.Wait()
-	return
+	group.Go(func() error {
+		info.Pathbuilders, _ = wisski.AllPathbuilders(server)
+		return nil
+	})
 }
 
 // Running checks if this WissKI is currently running.

internal/component/instances/wisski_pathbuilders.go

@@ -3,7 +3,6 @@ package instances
 import (
 	_ "embed"
 
-	"github.com/tkw1536/goprogram/stream"
 	"golang.org/x/exp/slices"
 )
 
@@ -11,21 +10,27 @@ import (
 var exportPathbuilderPHP string
 
 // Pathbuilders returns the ids of all pathbuilders in consistent order.
-func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
-	err = wisski.ExecPHPScript(stream.FromNil(), &ids, exportPathbuilderPHP, "all_list")
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (wisski *WissKI) Pathbuilders(server *PHPServer) (ids []string, err error) {
+	err = wisski.ExecPHPScript(server, &ids, exportPathbuilderPHP, "all_list")
 	slices.Sort(ids)
 	return
 }
 
 // Pathbuilder returns a single pathbuilder as xml.
 // If it does not exist, it returns the empty string and nil error.
-func (wisski *WissKI) Pathbuilder(id string) (xml string, err error) {
-	err = wisski.ExecPHPScript(stream.FromNil(), &xml, exportPathbuilderPHP, "one_xml", id)
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (wisski *WissKI) Pathbuilder(server *PHPServer, id string) (xml string, err error) {
+	err = wisski.ExecPHPScript(server, &xml, exportPathbuilderPHP, "one_xml", id)
 	return
 }
 
 // AllPathbuilders returns all pathbuilders serialized as xml
-func (wisski *WissKI) AllPathbuilders() (pathbuilders map[string]string, err error) {
-	err = wisski.ExecPHPScript(stream.FromNil(), &pathbuilders, exportPathbuilderPHP, "all_xml")
+//
+// server is the server to fetch the pathbuilders from, any may be nil.
+func (wisski *WissKI) AllPathbuilders(server *PHPServer) (pathbuilders map[string]string, err error) {
+	err = wisski.ExecPHPScript(server, &pathbuilders, exportPathbuilderPHP, "all_xml")
 	return
 }

internal/component/instances/wisski_php.go

@@ -1,141 +1,17 @@
 package instances
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"strings"
-
-	"github.com/tkw1536/goprogram/stream"
-
 	_ "embed"
 )
 
-var ErrExecInvalidCode = errors.New("invalid code to execute")
-var ErrExecNonZero = errors.New("script returned non-zero code")
-
-// ExecPHPScript executes the PHP code as a script within the wisski instance.
-// The script should define a function called entrypoint, and may define additional functions.
-//
-// Code must start with "<?php" and may not contain a closing tag.
-// Code is expected not to mess with PHPs output buffer.
-// Code should not contain user input.
-// Code breaking these conventions may or may not result in an error.
-//
-// It's arguments are encoded as json using [json.Marshal] and decoded within php.
-//
-// The return value of the function is again marshaled with json and returned to the caller.
-//
-// Standard input and output streams should not be used.
-// Standard error is redirected to io.
-func (wisski *WissKI) ExecPHPScript(io stream.IOStream, result any, code string, entrypoint string, args ...any) error {
-	// make sure the beginning is right
-	if !strings.HasPrefix(code, "<?php") {
-		return ErrExecInvalidCode
-	}
-
-	// make sure that args is not nil, but an array of length 0!
-	if args == nil {
-		args = []any{}
-	}
-
-	// encode code, args and entrypoint!
-	codeEscape, err := marshalPHP("?>" + code)
-	if err != nil {
-		return err
-	}
-
-	entrypointEscape, err := marshalPHP(entrypoint)
-	if err != nil {
-		return err
-	}
-
-	argsEscape, err := marshalPHP(args)
-	if err != nil {
-		return err
-	}
-
-	// assemble the script
-	script := `
-	ob_start(null, 0, PHP_OUTPUT_HANDLER_CLEANABLE);
-	eval(` + codeEscape + `);
-	ob_end_clean();
-
-	call_user_func(function(){
-		ob_start(null, 0, PHP_OUTPUT_HANDLER_CLEANABLE);
-		$result = call_user_func_array(` + entrypointEscape + `, ` + argsEscape + `);
-		ob_end_clean();
-		echo json_encode($result);
-	});
-`
-
-	// run the script
-	var output bytes.Buffer
-	res, err := wisski.Shell(io.Streams(&output, nil, strings.NewReader(script), 0), "-c", "drush php:script -")
-	if res != 0 {
-		return ErrExecNonZero
-	}
-	if err != nil {
-		return err
-	}
-
-	// did not request to receive a result
-	if result == nil {
-		return nil
-	}
-
-	// decode the output
-	return json.NewDecoder(&output).Decode(result)
-}
-
-const marshalRune = 'F' // press to pay respect
-
-// marshalPHP marshals some data which can be marshaled using [json.Encode] into a PHP Expression.
-// the string can be safely used directly within php.
-func marshalPHP(data any) (string, error) {
-	// this function uses json as a data format to transport the data into php.
-	// then we build a heredoc to encode it safely, and decode it in php
-
-	// Step 1: Encode the data as json
-	jbytes, err := json.Marshal(data)
-	if err != nil {
-		return "", err
-	}
-	jstring := string(jbytes)
-
-	// Step 2: Find a delimiter for the heredoc.
-	// Step 2a: Find the longest sequence of [marshalRune]s inside the encoded string.
-	var current, longest int
-	for _, r := range jstring {
-
-		if r == marshalRune {
-			current++
-		} else {
-			current = 0
-		}
-
-		if current > longest {
-			longest = current
-		}
-	}
-	// Step 2b: Build a string of marshalRune that is one longer!
-	delim := strings.Repeat(string(marshalRune), longest+1)
-
-	// Step 3: Assemble the encoded string!
-	result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
-	return result, nil
-}
-
-//
-
 //go:embed php/settings.php
 var settingsPHP string
 
-func (wisski *WissKI) GetSettingsPHP(key string) (value any, err error) {
-	err = wisski.ExecPHPScript(stream.FromNil(), &value, settingsPHP, "get_setting", key)
+func (wisski *WissKI) GetSettingsPHP(server *PHPServer, key string) (value any, err error) {
+	err = wisski.ExecPHPScript(server, &value, settingsPHP, "get_setting", key)
 	return
 }
 
-func (wisski *WissKI) SetSettingsPHP(key string, value any) error {
-	return wisski.ExecPHPScript(stream.FromNil(), nil, settingsPHP, "set_setting", key, value)
+func (wisski *WissKI) SetSettingsPHP(server *PHPServer, key string, value any) error {
+	return wisski.ExecPHPScript(server, nil, settingsPHP, "set_setting", key, value)
 }

internal/component/instances/wisski_php_server.go

@@ -0,0 +1,295 @@
+package instances
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+
+	_ "embed"
+
+	"github.com/alessio/shellescape"
+	"github.com/tkw1536/goprogram/lib/collection"
+	"github.com/tkw1536/goprogram/lib/nobufio"
+	"github.com/tkw1536/goprogram/stream"
+)
+
+// Common PHP Error
+var (
+	errPHPInit    = "Unable to initialize"
+	errPHPMarshal = "Marshal failed"
+	errPHPInvalid = PHPServerError{Message: "Invalid code to execute"}
+	errPHPReceive = "Failed to receive response"
+	errPHPClosed  = PHPServerError{Message: "Server closed"}
+)
+
+// PHPError represents an error during PHPServer logic
+type PHPServerError struct {
+	Message string
+	Err     error
+}
+
+func (err PHPServerError) Unwrap() error {
+	return err.Err
+}
+
+func (err PHPServerError) Error() string {
+	if err.Err == nil {
+		return fmt.Sprintf("PHPServer: %s", err.Message)
+	}
+	return fmt.Sprintf("PHPServer: %s: %s", err.Message, err.Err)
+}
+
+// PHPThrowable represents an error during php code
+type PHPThrowable string
+
+func (throwable PHPThrowable) Error() string {
+	return string(throwable)
+}
+
+// NewPHPServer returns a new server that can execute code within this distillery.
+// When err == nil, the caller must call server.Close().
+//
+// See [PHPServer].
+func (wisski *WissKI) NewPHPServer() (*PHPServer, error) {
+	// create input and output pipes
+	ir, iw, err := os.Pipe()
+	if err != nil {
+		return nil, PHPServerError{errPHPInit, err}
+	}
+	or, ow, err := os.Pipe()
+	if err != nil {
+		ir.Close()
+		iw.Close()
+		return nil, PHPServerError{errPHPInit, err}
+	}
+
+	// create a context to close the server
+	context, cancel := context.WithCancel(context.Background())
+
+	// start the shell process, which will close everything once done
+	go func() {
+		defer func() {
+			ir.Close()
+			iw.Close()
+			or.Close()
+			ow.Close()
+
+			cancel()
+		}()
+
+		// start the server
+		io := stream.NewIOStream(ow, nil, ir, 0)
+		wisski.Shell(io, "-c", shellescape.QuoteCommand([]string{"drush", "php:eval", serverPHP}))
+	}()
+
+	// return the seerver
+	return &PHPServer{
+		in:  iw,
+		out: or,
+		c:   context,
+	}, nil
+}
+
+// PHPServer represents a server that executes code within a distillery.
+// A typical use-case is to define functions using [MarshalEval], and then call those functions [MarshalCall].
+//
+// A nil PHPServer will return [ErrServerBroken] on every function call.
+type PHPServer struct {
+	m sync.Mutex
+
+	in  io.WriteCloser
+	out io.Reader
+	c   context.Context
+}
+
+// MarshalEval evaluates code on the server and Marshals the result into value.
+// When value is nil, the results are discarded.
+//
+// code is directly passed to php's "eval" function.
+// as such any functions defined will remain in server memory.
+//
+// When an exception is thrown by the PHP Code, error is not nil, and dest remains unchanged.
+func (server *PHPServer) MarshalEval(value any, code string) error {
+	server.m.Lock()
+	defer server.m.Unlock()
+
+	// quick hack: when the server is already done
+	if err := server.c.Err(); err != nil {
+		return errPHPClosed
+	}
+
+	// marshal the code, and send it to the server
+	bytes, err := json.Marshal(code)
+	if err != nil {
+		return PHPServerError{Message: errPHPMarshal, Err: err}
+	}
+
+	// send it to the server
+	io.WriteString(server.in, string(bytes)+"\n")
+
+	// read the next line (as a response)
+	data, err := nobufio.ReadLine(server.out)
+	if err != nil {
+		return PHPServerError{Message: errPHPReceive, Err: err}
+	}
+
+	// read whatever we received
+	var received [2]json.RawMessage
+	if err := json.Unmarshal([]byte(data), &received); err != nil {
+		return PHPServerError{Message: errPHPMarshal, Err: err}
+	}
+
+	// check if there was an error
+	var errString string
+	if err := json.Unmarshal(received[1], &errString); err == nil && errString != "" {
+		return PHPThrowable(errString)
+	}
+
+	// special case: no return value => no unmarshaling needed
+	if value == nil {
+		return nil
+	}
+
+	// read the actual result!
+	return json.Unmarshal(received[0], value)
+}
+
+// Eval is like [MarshalEval], but returns the value as an any
+func (server *PHPServer) Eval(code string) (value any, err error) {
+	err = server.MarshalEval(&value, code)
+	return
+}
+
+// MarshalCall calls a previously defined function with the given arguments.
+// Arguments are sent to php using json Marshal, and are 'json_decode'd on the php side.
+//
+// Return values are received as in [MarshalEval].
+func (server *PHPServer) MarshalCall(value any, function string, args ...any) error {
+	// marshal a code for the call
+	userFunction, err := marshalPHP(function)
+	if err != nil {
+		return PHPServerError{Message: errPHPMarshal, Err: err}
+	}
+	userFunctionArgs, err := marshalPHP(args)
+	if err != nil {
+		return PHPServerError{Message: errPHPMarshal, Err: err}
+	}
+	code := "return call_user_func_array(" + userFunction + "," + userFunctionArgs + ");"
+
+	// and return the evaluated code!
+	return server.MarshalEval(value, code)
+}
+
+// Call is like [MarshalCall] but returns the return value of the function as an any
+func (server *PHPServer) Call(function string, args ...any) (value any, err error) {
+	err = server.MarshalCall(&value, function, args...)
+	return
+}
+
+const marshalRune = 'F' // press to pay respect
+
+// marshalPHP marshals some data which can be marshaled using [json.Encode] into a PHP Expression.
+// the string can be safely used directly within php.
+func marshalPHP(data any) (string, error) {
+	// this function uses json as a data format to transport the data into php.
+	// then we build a heredoc to encode it safely, and decode it in php
+
+	// Step 1: Encode the data as json
+	jbytes, err := json.Marshal(data)
+	if err != nil {
+		return "", err
+	}
+	jstring := string(jbytes)
+
+	// Step 2: Find a delimiter for the heredoc.
+	// Step 2a: Find the longest sequence of [marshalRune]s inside the encoded string.
+	var current, longest int
+	for _, r := range jstring {
+
+		if r == marshalRune {
+			current++
+		} else {
+			current = 0
+		}
+
+		if current > longest {
+			longest = current
+		}
+	}
+	// Step 2b: Build a string of marshalRune that is one longer!
+	delim := strings.Repeat(string(marshalRune), longest+1)
+
+	// Step 3: Assemble the encoded string!
+	result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
+	return result, nil
+}
+
+// Close closes this server and prevents any further code from being run.
+func (server *PHPServer) Close() error {
+	server.m.Lock()
+	defer server.m.Unlock()
+
+	// if the context is already closed
+	if err := server.c.Err(); err != nil {
+		return errPHPClosed
+	}
+
+	server.in.Close()
+	<-server.c.Done()
+
+	return nil
+}
+
+// ExecPHPScript executes the PHP code as a script on the given server.
+// When server is nil, creates a new server and automatically closes it after execution.
+//
+// The script should define a function called entrypoint, and may define additional functions.
+//
+// Code must start with "<?php" and may not contain a closing tag.
+// Code is expected not to mess with PHPs output buffer.
+// Code should not contain user input.
+// Code breaking these conventions may or may not result in an error.
+//
+// It's arguments are encoded as json using [json.Marshal] and decoded within php.
+//
+// The return value of the function is again marshaled with json and returned to the caller.
+//
+// Calling this function is inefficient, and a [NewPHPServer] call should be prefered instead.
+func (wisski *WissKI) ExecPHPScript(server *PHPServer, value any, code string, entrypoint string, args ...any) (err error) {
+	if server == nil {
+		server, err = wisski.NewPHPServer()
+		if err != nil {
+			return
+		}
+		defer server.Close()
+	}
+
+	if err := server.MarshalEval(nil, strings.TrimPrefix(code, "<?php")); err != nil {
+		return err
+	}
+
+	return server.MarshalCall(value, entrypoint, args...)
+}
+
+//go:embed php/server.php
+var serverPHP string
+
+// pre-process the server.php code to make it shorter
+func init() {
+	// remove the first '<?php' line
+	lines := strings.Split(serverPHP, "\n")[1:]
+	for i, line := range lines {
+		lines[i] = strings.TrimSpace(line)
+	}
+
+	// remove comment lines
+	lines = collection.Filter(lines, func(line string) bool {
+		return !strings.HasPrefix(line, "//")
+	})
+
+	serverPHP = strings.Join(lines, "")
+}

internal/component/instances/wisski_prefix.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
 	"github.com/tkw1536/goprogram/lib/collection"
-	"github.com/tkw1536/goprogram/stream"
 
 	_ "embed"
 )
@@ -22,8 +21,11 @@ func (wisski *WissKI) NoPrefix() bool {
 var listURIPrefixesPHP string
 
 // Prefixes returns the prefixes applying to this WissKI
-func (wisski *WissKI) Prefixes() ([]string, error) {
-	prefixes, err := wisski.dbPrefixes()
+//
+// server is an optional server to fetch prefixes from.
+// server may be nil.
+func (wisski *WissKI) Prefixes(server *PHPServer) ([]string, error) {
+	prefixes, err := wisski.dbPrefixes(server)
 	if err != nil {
 		return nil, err
 	}
@@ -36,9 +38,9 @@ func (wisski *WissKI) Prefixes() ([]string, error) {
 	return append(prefixes, prefixes2...), nil
 }
 
-func (wisski *WissKI) dbPrefixes() (prefixes []string, err error) {
+func (wisski *WissKI) dbPrefixes(server *PHPServer) (prefixes []string, err error) {
 	// get all the ugly prefixes
-	err = wisski.ExecPHPScript(stream.FromNil(), &prefixes, listURIPrefixesPHP, "list_prefixes")
+	err = wisski.ExecPHPScript(server, &prefixes, listURIPrefixesPHP, "list_prefixes")
 	if err != nil {
 		return nil, err
 	}
@@ -139,7 +141,7 @@ func (wisski *WissKI) PrefixesCached() (results []string, err error) {
 
 // UpdatePrefixes updates the cached prefixes of this instance
 func (wisski *WissKI) UpdatePrefixes() error {
-	prefixes, err := wisski.Prefixes()
+	prefixes, err := wisski.Prefixes(nil)
 	if err != nil {
 		return err
 	}

internal/component/snapshots/extras_pathbuilders.go

@@ -21,7 +21,7 @@ func (Pathbuilders) SnapshotName() string { return "pathbuilders" }
 
 func (pbs *Pathbuilders) Snapshot(wisski models.Instance, context component.StagingContext) error {
 	return context.AddDirectory(".", func() error {
-		builders, err := pbs.Instances.Instance(wisski).AllPathbuilders()
+		builders, err := pbs.Instances.Instance(wisski).AllPathbuilders(nil)
 		if err != nil {
 			return err
 		}