add wisski_php_server

This commit is contained in:
Tom Wiesing 2022-10-16 11:32:03 +02:00
parent 45af2cc95b
commit c258b46443
No known key found for this signature in database
11 changed files with 411 additions and 183 deletions

View file

@ -47,7 +47,7 @@ func (ds setting) Run(context wisski_distillery.Context) error {
if ds.Positionals.Value == "" {
// get the setting
value, err := instance.GetSettingsPHP(ds.Positionals.Setting)
value, err := instance.GetSettingsPHP(nil, ds.Positionals.Setting)
if err != nil {
return errSettingGet.Wrap(err)
}
@ -69,7 +69,7 @@ func (ds setting) Run(context wisski_distillery.Context) error {
}
// set the serialized value!
if err := instance.SetSettingsPHP(ds.Positionals.Setting, data); err != nil {
if err := instance.SetSettingsPHP(nil, ds.Positionals.Setting, data); err != nil {
return errSettingSet.Wrap(err)
}

View file

@ -60,7 +60,7 @@ func (i info) Run(context wisski_distillery.Context) error {
context.Printf("Running: %v\n", info.Running)
context.Printf("Locked: %v\n", info.Locked)
context.Printf("Last Rebuild: %v\n", info.LastRebuild.String())
context.Printf("Last Update: %v\n", info.LastUpdate.String())
context.Printf("Last Update: %v\n", info.LastUpdate.String())
context.Printf("Skip Prefixes: %v\n", info.NoPrefixes)
context.Printf("Prefixes: (count %d)\n", len(info.Prefixes))

View file

@ -37,6 +37,7 @@ var errNoPathbuilder = exit.Error{
}
func (pb pathbuilders) Run(context wisski_distillery.Context) error {
// get the wisski
instance, err := context.Environment.Instances().WissKI(pb.Positionals.Slug)
if err != nil {
@ -45,7 +46,7 @@ func (pb pathbuilders) Run(context wisski_distillery.Context) error {
// get all of the pathbuilders
if pb.Positionals.Name == "" {
names, err := instance.Pathbuilders()
names, err := instance.Pathbuilders(nil)
if err != nil {
return errPathbuilders.WithMessageF(err)
}
@ -56,7 +57,7 @@ func (pb pathbuilders) Run(context wisski_distillery.Context) error {
}
// get all the pathbuilders
xml, err := instance.Pathbuilder(pb.Positionals.Name)
xml, err := instance.Pathbuilder(nil, pb.Positionals.Name)
if xml == "" {
return errNoPathbuilder.WithMessageF(pb.Positionals.Name)
}

View file

@ -36,7 +36,7 @@ func (p prefixes) Run(context wisski_distillery.Context) error {
return err
}
prefixes, err := instance.Prefixes()
prefixes, err := instance.Prefixes(nil)
if err != nil {
return errPrefixesGeneric.Wrap(err)
}

View file

@ -0,0 +1,34 @@
<?php
// This file contains code to execute a php execution server.
// It is passed as a *command line literal * directly to 'drush:script'.
//
// As such it is preprocessed and shortened.
// It should only contain comments at the beginning of each line, and only starting with '//'.
// See wisski_php_server.go.
// don't buffer stdin!
stream_set_read_buffer(STDIN,0);
// stop all other output
ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
while($line = fgets(STDIN)){
// decode the command to run
$code=@json_decode($line);
// execute it
try{
$json = json_encode([eval($code),""]);
}catch(Throwable $t){
$json = json_encode([null,(string)$t]);
}
if($json===false) {
$json = '[null,"Error encoding result"]';
}
// and write out the result
ob_end_clean();
fwrite(STDOUT,"$json\n");
ob_start(null,0,PHP_OUTPUT_HANDLER_CLEANABLE);
}

View file

@ -34,57 +34,72 @@ type WissKIInfo struct {
// Info generate a
func (wisski *WissKI) Info(quick bool) (info WissKIInfo, err error) {
// TODO: Cache this, and run it with every cron!
var group errgroup.Group
wisski.infoQuick(&info, &group)
if !quick {
server, err := wisski.NewPHPServer()
if err == nil {
wisski.infoSlow(&info, server, &group)
defer server.Close()
}
}
err = group.Wait()
return
}
func (wisski *WissKI) infoQuick(info *WissKIInfo, group *errgroup.Group) {
info.Time = time.Now().UTC()
// static properties
info.Slug = wisski.Slug
info.URL = wisski.URL().String()
// dynamic properties
var group errgroup.Group
// quick check if this wisski is running
group.Go(func() (err error) {
info.Running, err = wisski.Running()
return
})
// quick check if this instance is locked
group.Go(func() (err error) {
info.Locked = wisski.IsLocked()
return
})
group.Go(func() (err error) {
info.LastRebuild, _ = wisski.LastRebuild()
return
})
group.Go(func() (err error) {
info.LastUpdate, _ = wisski.LastUpdate()
return
})
group.Go(func() (err error) {
info.LastRebuild, _ = wisski.LastRebuild()
return
})
group.Go(func() (err error) {
info.NoPrefixes = wisski.NoPrefix()
return
})
}
func (wisski *WissKI) infoSlow(info *WissKIInfo, server *PHPServer, group *errgroup.Group) {
group.Go(func() (err error) {
info.Prefixes, _ = wisski.Prefixes(server)
return nil
})
// slower checks for extra properties.
// these might execute php code or require additional database queries.
if !quick {
group.Go(func() (err error) {
info.LastRebuild, _ = wisski.LastRebuild()
return nil
})
group.Go(func() (err error) {
info.LastUpdate, _ = wisski.LastUpdate()
return nil
})
group.Go(func() error {
info.Pathbuilders, _ = wisski.AllPathbuilders()
return nil
})
group.Go(func() (err error) {
info.Prefixes, _ = wisski.Prefixes()
info.NoPrefixes = wisski.NoPrefix()
return nil
})
group.Go(func() error {
info.Snapshots, _ = wisski.Snapshots()
return nil
})
}
group.Go(func() error {
info.Snapshots, _ = wisski.Snapshots()
return nil
})
err = group.Wait()
return
group.Go(func() error {
info.Pathbuilders, _ = wisski.AllPathbuilders(server)
return nil
})
}
// Running checks if this WissKI is currently running.

View file

@ -3,7 +3,6 @@ package instances
import (
_ "embed"
"github.com/tkw1536/goprogram/stream"
"golang.org/x/exp/slices"
)
@ -11,21 +10,27 @@ import (
var exportPathbuilderPHP string
// Pathbuilders returns the ids of all pathbuilders in consistent order.
func (wisski *WissKI) Pathbuilders() (ids []string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &ids, exportPathbuilderPHP, "all_list")
//
// server is the server to fetch the pathbuilders from, any may be nil.
func (wisski *WissKI) Pathbuilders(server *PHPServer) (ids []string, err error) {
err = wisski.ExecPHPScript(server, &ids, exportPathbuilderPHP, "all_list")
slices.Sort(ids)
return
}
// Pathbuilder returns a single pathbuilder as xml.
// If it does not exist, it returns the empty string and nil error.
func (wisski *WissKI) Pathbuilder(id string) (xml string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &xml, exportPathbuilderPHP, "one_xml", id)
//
// server is the server to fetch the pathbuilders from, any may be nil.
func (wisski *WissKI) Pathbuilder(server *PHPServer, id string) (xml string, err error) {
err = wisski.ExecPHPScript(server, &xml, exportPathbuilderPHP, "one_xml", id)
return
}
// AllPathbuilders returns all pathbuilders serialized as xml
func (wisski *WissKI) AllPathbuilders() (pathbuilders map[string]string, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &pathbuilders, exportPathbuilderPHP, "all_xml")
//
// server is the server to fetch the pathbuilders from, any may be nil.
func (wisski *WissKI) AllPathbuilders(server *PHPServer) (pathbuilders map[string]string, err error) {
err = wisski.ExecPHPScript(server, &pathbuilders, exportPathbuilderPHP, "all_xml")
return
}

View file

@ -1,141 +1,17 @@
package instances
import (
"bytes"
"encoding/json"
"errors"
"strings"
"github.com/tkw1536/goprogram/stream"
_ "embed"
)
var ErrExecInvalidCode = errors.New("invalid code to execute")
var ErrExecNonZero = errors.New("script returned non-zero code")
// ExecPHPScript executes the PHP code as a script within the wisski instance.
// The script should define a function called entrypoint, and may define additional functions.
//
// Code must start with "<?php" and may not contain a closing tag.
// Code is expected not to mess with PHPs output buffer.
// Code should not contain user input.
// Code breaking these conventions may or may not result in an error.
//
// It's arguments are encoded as json using [json.Marshal] and decoded within php.
//
// The return value of the function is again marshaled with json and returned to the caller.
//
// Standard input and output streams should not be used.
// Standard error is redirected to io.
func (wisski *WissKI) ExecPHPScript(io stream.IOStream, result any, code string, entrypoint string, args ...any) error {
// make sure the beginning is right
if !strings.HasPrefix(code, "<?php") {
return ErrExecInvalidCode
}
// make sure that args is not nil, but an array of length 0!
if args == nil {
args = []any{}
}
// encode code, args and entrypoint!
codeEscape, err := marshalPHP("?>" + code)
if err != nil {
return err
}
entrypointEscape, err := marshalPHP(entrypoint)
if err != nil {
return err
}
argsEscape, err := marshalPHP(args)
if err != nil {
return err
}
// assemble the script
script := `
ob_start(null, 0, PHP_OUTPUT_HANDLER_CLEANABLE);
eval(` + codeEscape + `);
ob_end_clean();
call_user_func(function(){
ob_start(null, 0, PHP_OUTPUT_HANDLER_CLEANABLE);
$result = call_user_func_array(` + entrypointEscape + `, ` + argsEscape + `);
ob_end_clean();
echo json_encode($result);
});
`
// run the script
var output bytes.Buffer
res, err := wisski.Shell(io.Streams(&output, nil, strings.NewReader(script), 0), "-c", "drush php:script -")
if res != 0 {
return ErrExecNonZero
}
if err != nil {
return err
}
// did not request to receive a result
if result == nil {
return nil
}
// decode the output
return json.NewDecoder(&output).Decode(result)
}
const marshalRune = 'F' // press to pay respect
// marshalPHP marshals some data which can be marshaled using [json.Encode] into a PHP Expression.
// the string can be safely used directly within php.
func marshalPHP(data any) (string, error) {
// this function uses json as a data format to transport the data into php.
// then we build a heredoc to encode it safely, and decode it in php
// Step 1: Encode the data as json
jbytes, err := json.Marshal(data)
if err != nil {
return "", err
}
jstring := string(jbytes)
// Step 2: Find a delimiter for the heredoc.
// Step 2a: Find the longest sequence of [marshalRune]s inside the encoded string.
var current, longest int
for _, r := range jstring {
if r == marshalRune {
current++
} else {
current = 0
}
if current > longest {
longest = current
}
}
// Step 2b: Build a string of marshalRune that is one longer!
delim := strings.Repeat(string(marshalRune), longest+1)
// Step 3: Assemble the encoded string!
result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
return result, nil
}
//
//go:embed php/settings.php
var settingsPHP string
func (wisski *WissKI) GetSettingsPHP(key string) (value any, err error) {
err = wisski.ExecPHPScript(stream.FromNil(), &value, settingsPHP, "get_setting", key)
func (wisski *WissKI) GetSettingsPHP(server *PHPServer, key string) (value any, err error) {
err = wisski.ExecPHPScript(server, &value, settingsPHP, "get_setting", key)
return
}
func (wisski *WissKI) SetSettingsPHP(key string, value any) error {
return wisski.ExecPHPScript(stream.FromNil(), nil, settingsPHP, "set_setting", key, value)
func (wisski *WissKI) SetSettingsPHP(server *PHPServer, key string, value any) error {
return wisski.ExecPHPScript(server, nil, settingsPHP, "set_setting", key, value)
}

View file

@ -0,0 +1,295 @@
package instances
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"sync"
_ "embed"
"github.com/alessio/shellescape"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/lib/nobufio"
"github.com/tkw1536/goprogram/stream"
)
// Common PHP Error
var (
errPHPInit = "Unable to initialize"
errPHPMarshal = "Marshal failed"
errPHPInvalid = PHPServerError{Message: "Invalid code to execute"}
errPHPReceive = "Failed to receive response"
errPHPClosed = PHPServerError{Message: "Server closed"}
)
// PHPError represents an error during PHPServer logic
type PHPServerError struct {
Message string
Err error
}
func (err PHPServerError) Unwrap() error {
return err.Err
}
func (err PHPServerError) Error() string {
if err.Err == nil {
return fmt.Sprintf("PHPServer: %s", err.Message)
}
return fmt.Sprintf("PHPServer: %s: %s", err.Message, err.Err)
}
// PHPThrowable represents an error during php code
type PHPThrowable string
func (throwable PHPThrowable) Error() string {
return string(throwable)
}
// NewPHPServer returns a new server that can execute code within this distillery.
// When err == nil, the caller must call server.Close().
//
// See [PHPServer].
func (wisski *WissKI) NewPHPServer() (*PHPServer, error) {
// create input and output pipes
ir, iw, err := os.Pipe()
if err != nil {
return nil, PHPServerError{errPHPInit, err}
}
or, ow, err := os.Pipe()
if err != nil {
ir.Close()
iw.Close()
return nil, PHPServerError{errPHPInit, err}
}
// create a context to close the server
context, cancel := context.WithCancel(context.Background())
// start the shell process, which will close everything once done
go func() {
defer func() {
ir.Close()
iw.Close()
or.Close()
ow.Close()
cancel()
}()
// start the server
io := stream.NewIOStream(ow, nil, ir, 0)
wisski.Shell(io, "-c", shellescape.QuoteCommand([]string{"drush", "php:eval", serverPHP}))
}()
// return the seerver
return &PHPServer{
in: iw,
out: or,
c: context,
}, nil
}
// PHPServer represents a server that executes code within a distillery.
// A typical use-case is to define functions using [MarshalEval], and then call those functions [MarshalCall].
//
// A nil PHPServer will return [ErrServerBroken] on every function call.
type PHPServer struct {
m sync.Mutex
in io.WriteCloser
out io.Reader
c context.Context
}
// MarshalEval evaluates code on the server and Marshals the result into value.
// When value is nil, the results are discarded.
//
// code is directly passed to php's "eval" function.
// as such any functions defined will remain in server memory.
//
// When an exception is thrown by the PHP Code, error is not nil, and dest remains unchanged.
func (server *PHPServer) MarshalEval(value any, code string) error {
server.m.Lock()
defer server.m.Unlock()
// quick hack: when the server is already done
if err := server.c.Err(); err != nil {
return errPHPClosed
}
// marshal the code, and send it to the server
bytes, err := json.Marshal(code)
if err != nil {
return PHPServerError{Message: errPHPMarshal, Err: err}
}
// send it to the server
io.WriteString(server.in, string(bytes)+"\n")
// read the next line (as a response)
data, err := nobufio.ReadLine(server.out)
if err != nil {
return PHPServerError{Message: errPHPReceive, Err: err}
}
// read whatever we received
var received [2]json.RawMessage
if err := json.Unmarshal([]byte(data), &received); err != nil {
return PHPServerError{Message: errPHPMarshal, Err: err}
}
// check if there was an error
var errString string
if err := json.Unmarshal(received[1], &errString); err == nil && errString != "" {
return PHPThrowable(errString)
}
// special case: no return value => no unmarshaling needed
if value == nil {
return nil
}
// read the actual result!
return json.Unmarshal(received[0], value)
}
// Eval is like [MarshalEval], but returns the value as an any
func (server *PHPServer) Eval(code string) (value any, err error) {
err = server.MarshalEval(&value, code)
return
}
// MarshalCall calls a previously defined function with the given arguments.
// Arguments are sent to php using json Marshal, and are 'json_decode'd on the php side.
//
// Return values are received as in [MarshalEval].
func (server *PHPServer) MarshalCall(value any, function string, args ...any) error {
// marshal a code for the call
userFunction, err := marshalPHP(function)
if err != nil {
return PHPServerError{Message: errPHPMarshal, Err: err}
}
userFunctionArgs, err := marshalPHP(args)
if err != nil {
return PHPServerError{Message: errPHPMarshal, Err: err}
}
code := "return call_user_func_array(" + userFunction + "," + userFunctionArgs + ");"
// and return the evaluated code!
return server.MarshalEval(value, code)
}
// Call is like [MarshalCall] but returns the return value of the function as an any
func (server *PHPServer) Call(function string, args ...any) (value any, err error) {
err = server.MarshalCall(&value, function, args...)
return
}
const marshalRune = 'F' // press to pay respect
// marshalPHP marshals some data which can be marshaled using [json.Encode] into a PHP Expression.
// the string can be safely used directly within php.
func marshalPHP(data any) (string, error) {
// this function uses json as a data format to transport the data into php.
// then we build a heredoc to encode it safely, and decode it in php
// Step 1: Encode the data as json
jbytes, err := json.Marshal(data)
if err != nil {
return "", err
}
jstring := string(jbytes)
// Step 2: Find a delimiter for the heredoc.
// Step 2a: Find the longest sequence of [marshalRune]s inside the encoded string.
var current, longest int
for _, r := range jstring {
if r == marshalRune {
current++
} else {
current = 0
}
if current > longest {
longest = current
}
}
// Step 2b: Build a string of marshalRune that is one longer!
delim := strings.Repeat(string(marshalRune), longest+1)
// Step 3: Assemble the encoded string!
result := "call_user_func(function(){$x=<<<'" + delim + "'\n" + jstring + "\n" + delim + ";return json_decode(trim($x));})" // press to doubt
return result, nil
}
// Close closes this server and prevents any further code from being run.
func (server *PHPServer) Close() error {
server.m.Lock()
defer server.m.Unlock()
// if the context is already closed
if err := server.c.Err(); err != nil {
return errPHPClosed
}
server.in.Close()
<-server.c.Done()
return nil
}
// ExecPHPScript executes the PHP code as a script on the given server.
// When server is nil, creates a new server and automatically closes it after execution.
//
// The script should define a function called entrypoint, and may define additional functions.
//
// Code must start with "<?php" and may not contain a closing tag.
// Code is expected not to mess with PHPs output buffer.
// Code should not contain user input.
// Code breaking these conventions may or may not result in an error.
//
// It's arguments are encoded as json using [json.Marshal] and decoded within php.
//
// The return value of the function is again marshaled with json and returned to the caller.
//
// Calling this function is inefficient, and a [NewPHPServer] call should be prefered instead.
func (wisski *WissKI) ExecPHPScript(server *PHPServer, value any, code string, entrypoint string, args ...any) (err error) {
if server == nil {
server, err = wisski.NewPHPServer()
if err != nil {
return
}
defer server.Close()
}
if err := server.MarshalEval(nil, strings.TrimPrefix(code, "<?php")); err != nil {
return err
}
return server.MarshalCall(value, entrypoint, args...)
}
//go:embed php/server.php
var serverPHP string
// pre-process the server.php code to make it shorter
func init() {
// remove the first '<?php' line
lines := strings.Split(serverPHP, "\n")[1:]
for i, line := range lines {
lines[i] = strings.TrimSpace(line)
}
// remove comment lines
lines = collection.Filter(lines, func(line string) bool {
return !strings.HasPrefix(line, "//")
})
serverPHP = strings.Join(lines, "")
}

View file

@ -7,7 +7,6 @@ import (
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/tkw1536/goprogram/lib/collection"
"github.com/tkw1536/goprogram/stream"
_ "embed"
)
@ -22,8 +21,11 @@ func (wisski *WissKI) NoPrefix() bool {
var listURIPrefixesPHP string
// Prefixes returns the prefixes applying to this WissKI
func (wisski *WissKI) Prefixes() ([]string, error) {
prefixes, err := wisski.dbPrefixes()
//
// server is an optional server to fetch prefixes from.
// server may be nil.
func (wisski *WissKI) Prefixes(server *PHPServer) ([]string, error) {
prefixes, err := wisski.dbPrefixes(server)
if err != nil {
return nil, err
}
@ -36,9 +38,9 @@ func (wisski *WissKI) Prefixes() ([]string, error) {
return append(prefixes, prefixes2...), nil
}
func (wisski *WissKI) dbPrefixes() (prefixes []string, err error) {
func (wisski *WissKI) dbPrefixes(server *PHPServer) (prefixes []string, err error) {
// get all the ugly prefixes
err = wisski.ExecPHPScript(stream.FromNil(), &prefixes, listURIPrefixesPHP, "list_prefixes")
err = wisski.ExecPHPScript(server, &prefixes, listURIPrefixesPHP, "list_prefixes")
if err != nil {
return nil, err
}
@ -139,7 +141,7 @@ func (wisski *WissKI) PrefixesCached() (results []string, err error) {
// UpdatePrefixes updates the cached prefixes of this instance
func (wisski *WissKI) UpdatePrefixes() error {
prefixes, err := wisski.Prefixes()
prefixes, err := wisski.Prefixes(nil)
if err != nil {
return err
}

View file

@ -21,7 +21,7 @@ func (Pathbuilders) SnapshotName() string { return "pathbuilders" }
func (pbs *Pathbuilders) Snapshot(wisski models.Instance, context component.StagingContext) error {
return context.AddDirectory(".", func() error {
builders, err := pbs.Instances.Instance(wisski).AllPathbuilders()
builders, err := pbs.Instances.Instance(wisski).AllPathbuilders(nil)
if err != nil {
return err
}