rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

triplestore: Use http timeout if possible

Browse source
This commit is contained in:
Tom Wiesing 2023-11-15 09:51:35 +01:00
parent 6a739df24b
commit aacde06636
No known key found for this signature in database
8 changed files with 170 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

2
internal/dis/component/triplestore/backup.go
View file

@ -32,7 +32,7 @@ func (ts *Triplestore) Backup(scontext *component.StagingContext) error {
}
func (ts Triplestore) listRepositories(ctx context.Context) (repos []Repository, err error) {
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "application/json")
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "application/json", 0)
if err != nil {
return nil, err
}

13
internal/dis/component/triplestore/database.go
View file

@ -27,11 +27,15 @@ type TriplestoreUserAppSettings struct {
ExecuteCount bool `json:"EXECUTE_COUNT"`
}
// http.Client Timeout to be used for "trivial" triplestore operations.
// This includes e.g. CRUDing a specific repo.
const tsTrivialTimeout = time.Minute
// OpenRaw makes an http request to the triplestore api.
//
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
// When bodyName is empty, simply marshal body as application/json
func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any, bodyName string, accept string) (*http.Response, error) {
func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any, bodyName string, accept string, timeout time.Duration) (*http.Response, error) {
var reader io.Reader // to read the body from
var contentType string // content-type of the request being sent
@ -66,6 +70,7 @@ func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any,
// create the request object
client := &http.Client{
Timeout: timeout,
Transport: &http.Transport{
DisableKeepAlives: true,
},
@ -92,7 +97,7 @@ func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any,
// This is achieved using a polling strategy.
func (ts Triplestore) Wait(ctx context.Context) error {
return timex.TickUntilFunc(func(time.Time) bool {
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "")
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "", tsTrivialTimeout)
zerolog.Ctx(ctx).Trace().Err(err).Msg("Triplestore wait")
if err != nil {
return false
@ -105,7 +110,7 @@ func (ts Triplestore) Wait(ctx context.Context) error {
// PurgeUser deletes the specified user from the triplestore.
// When the user does not exist, returns no error.
func (ts Triplestore) PurgeUser(ctx context.Context, user string) error {
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/security/users/"+user, nil, "", "")
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/security/users/"+user, nil, "", "", tsTrivialTimeout)
if err != nil {
return err
}
@ -118,7 +123,7 @@ func (ts Triplestore) PurgeUser(ctx context.Context, user string) error {
// PurgeRepo deletes the specified repo from the triplestore.
// When the repo does not exist, returns no error.
func (ts Triplestore) PurgeRepo(ctx context.Context, repo string) error {
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/repositories/"+repo, nil, "", "")
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/repositories/"+repo, nil, "", "", tsTrivialTimeout)
if err != nil {
return err
}

4
internal/dis/component/triplestore/provision.go
View file

@ -61,7 +61,7 @@ func (ts *Triplestore) CreateRepository(ctx context.Context, name, domain, user,
// do the create!
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/repositories", createRepo.Bytes(), "config", "")
res, err := ts.OpenRaw(ctx, "POST", "/rest/repositories", createRepo.Bytes(), "config", "", tsTrivialTimeout)
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}
@ -87,7 +87,7 @@ func (ts *Triplestore) CreateRepository(ctx context.Context, name, domain, user,
"READ_REPO_" + name,
"WRITE_REPO_" + name,
},
}, "", "")
}, "", "", tsTrivialTimeout)
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}

2
internal/dis/component/triplestore/snapshot.go
View file

@ -27,7 +27,7 @@ var errTSBackupWrongStatusCode = errors.New("Triplestore.Backup: Wrong status co
// SnapshotDB snapshots the provided repository into dst
func (ts Triplestore) SnapshotDB(ctx context.Context, dst io.Writer, repo string) (int64, error) {
res, err := ts.OpenRaw(ctx, "GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads")
res, err := ts.OpenRaw(ctx, "GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads", 0)
if err != nil {
return 0, err
}

4
internal/dis/component/triplestore/update.go
View file

@ -30,7 +30,7 @@ func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
ExecuteCount: true,
},
GrantedAuthorities: []string{"ROLE_ADMIN"},
}, "", "")
}, "", "", tsTrivialTimeout)
if err != nil {
return fmt.Errorf("failed to create triplestore user: %s", err)
}
@ -52,7 +52,7 @@ func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
logging.LogMessage(progress, "Enabling Triplestore security")
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/security", true, "", "")
res, err := ts.OpenRaw(ctx, "POST", "/rest/security", true, "", "", tsTrivialTimeout)
if err != nil {
return fmt.Errorf("failed to enable triplestore security: %s", err)
}

127
internal/phpx/marshal.go
View file

@ -2,10 +2,14 @@ package phpx
import (
"encoding/json"
"math"
"strconv"
"strings"
"github.com/tkw1536/pkglib/collection"
)
// Marshal marshals data as a PHP expression, meaning it can safely be used inside code.
// Marshal marshals data as a PHP expression, so that it can be safely used inside a php expession.
//
// Typically data is marshaled using [json.Marshal] and decoded in PHP using 'json_decode'.
// Special cases may exist for specific datatypes.
@ -23,11 +27,124 @@ func Marshal(data any) (string, error) {
return "json_decode(" + MarshalString(string(bytes)) + ")", nil
}
var replacer = strings.NewReplacer("'", "\\'", "\\", "\\\\")
// MarshalJSON marshals a json-value safely as an expression to be used as a php string.
//
// A json value is one returned by calling [json.Unmarshal] on a value of type any.
// These are then marshaled by the appropriate function:
//
// - a nil is turned into [PHPNil]
// - a bool is passed to [MarshalBool]
// - a float64 is passed to [MarshalFloat]
// - a string is passed to [MarshalString]
// - an []any is passed to [MarshalSlice]
// - an map[string]any is passed to [MarshalMap]
//
// All marshaling attempts to minify the length of the returned string, meaning compact encodings
// are prefered over length ones.
//
// If a value is none of these types, an empty string is returned.
// No valid value ever returns the empty string
func MarshalJSON(v any) string {
switch v := v.(type) {
case nil:
return PHPNil
case bool:
return MarshalBool(v)
case float64:
return MarshalFloat(v)
case string:
return MarshalString(v)
case []any:
return MarshalSlice(v)
case map[string]any:
return MarshalMap(v)
}
return ""
}
const (
// PHPNil represents the equivalent of a nil value in php
PHPNil = "null"
phpTrue = "!0"
phpFalse = "!1"
phpNaN = "NAN"
phpPositiveInfinity = "INF"
phpNegativeInfinity = "-INF"
)
// MarshalBool marshals b as a boolean to be used in php code.
// This corresponds to the strings "true" or "false".
func MarshalBool(b bool) string {
if b {
return phpTrue
}
return phpFalse
}
// MarshalFloat marshals a floating point number or integer
func MarshalFloat(f float64) string {
// if we actually have an integer, return it!
if i := int64(f); f == float64(i) {
return MarshalInt(i)
}
// special cases
if math.IsNaN(f) {
return phpNaN
}
if math.IsInf(f, 1) {
return phpPositiveInfinity
}
if math.IsInf(f, -1) {
return phpNegativeInfinity
}
// all other cases
return strconv.FormatFloat(f, 'E', -1, 64)
}
// MarshalInt marshals an integer as a string to be used inside a php literal
func MarshalInt(i int64) string {
return strconv.FormatInt(i, 10)
}
var stringReplacer = strings.NewReplacer("'", "\\'", "\\", "\\\\")
// MarshalString marshals s as a php string that can be used safely as a PHP expression.
//
// See [https://www.php.net/manual/en/language.types.string.php#language.types.string.syntax.single].
func MarshalString(s string) string {
return "'" + replacer.Replace(s) + "'"
// See [https://www.php.net/manual/en/language.types.string.php#language.types.string.syntax.single]
// we just escape
return "'" + stringReplacer.Replace(s) + "'"
}
func MarshalSlice(slice []any) string {
var builder strings.Builder
builder.WriteRune('[')
{
for _, v := range slice {
builder.WriteString(MarshalJSON(v))
builder.WriteRune(',')
}
}
builder.WriteRune(']')
return builder.String()
}
func MarshalMap(m map[string]any) string {
var builder strings.Builder
builder.WriteString("array(")
collection.IterateSorted(m, func(k string, v any) {
builder.WriteString(MarshalString(k))
builder.WriteString("=>")
builder.WriteString(MarshalJSON(v))
builder.WriteString(",")
})
builder.WriteString(")")
return builder.String()
}