rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add initial triplestore rebuild functionality

Browse source
This commit is contained in:
Tom Wiesing 2024-04-04 13:52:15 +02:00
parent 8a1319df16
commit 674b9d8d07
No known key found for this signature in database
11 changed files with 252 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

3
internal/dis/component/triplestore/backup.go
View file

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"io"
"net/http"
"github.com/FAU-CDI/wisski-distillery/internal/dis/component"
)
@ -32,7 +33,7 @@ func (ts *Triplestore) Backup(scontext *component.StagingContext) error {
}
func (ts Triplestore) listRepositories(ctx context.Context) (repos []Repository, err error) {
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "application/json", 0)
res, err := ts.DoRest(ctx, 0, http.MethodGet, "/rest/repositories", &RequestHeaders{Accept: "application/json"})
if err != nil {
return nil, err
}

122
internal/dis/component/triplestore/database.go
View file

@ -31,43 +31,71 @@ type TriplestoreUserAppSettings struct {
// This includes e.g. CRUDing a specific repo.
const tsTrivialTimeout = time.Minute
// OpenRaw makes an http request to the triplestore api.
//
// When bodyName is non-empty, expect body to be a byte slice representing a multipart/form-data upload with the given name.
// When bodyName is empty, simply marshal body as application/json
func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any, bodyName string, accept string, timeout time.Duration) (*http.Response, error) {
var reader io.Reader // to read the body from
var contentType string // content-type of the request being sent
// RequestHeaders represent headers of a raw http request
type RequestHeaders struct {
Accept string
ContentType string
}
// for "PUT" and "POST" we setup a body
if method == http.MethodPut || method == http.MethodPost {
if bodyName != "" {
var buffer bytes.Buffer
func (rh *RequestHeaders) With(headers RequestHeaders) *RequestHeaders {
// write the file to it
writer := multipart.NewWriter(&buffer)
{
part, err := writer.CreateFormFile(bodyName, "filename.txt")
if err != nil {
return nil, err
}
io.Copy(part, bytes.NewReader(body.([]byte)))
}
writer.Close()
// use it for the request
reader = &buffer
contentType = writer.FormDataContentType()
} else {
mbytes, err := json.Marshal(body)
if err != nil {
return nil, err
}
reader = bytes.NewReader(mbytes)
contentType = "application/json"
}
// create new request headers and copy the old options
var newHeaders RequestHeaders
if rh != nil {
newHeaders = *rh
}
// add the options
if headers.Accept != "" {
newHeaders.Accept = headers.Accept
}
if headers.ContentType != "" {
newHeaders.ContentType = headers.ContentType
}
return &newHeaders
}
// DoRest performs a (raw) http request to the without a body.
func (ts Triplestore) DoRest(ctx context.Context, timeout time.Duration, method, url string, headers *RequestHeaders) (*http.Response, error) {
return ts.DoRestWithReader(ctx, timeout, method, url, headers, nil)
}
// DoRestWithForm performs a http request where the body are all bytes read from fieldvalue.
func (ts Triplestore) DoRestWithForm(ctx context.Context, timeout time.Duration, method, url string, headers *RequestHeaders, fieldname string, fieldvalue io.Reader) (*http.Response, error) {
var buffer bytes.Buffer
// write the file to it
writer := multipart.NewWriter(&buffer)
{
part, err := writer.CreateFormFile(fieldname, "filename.txt")
if err != nil {
return nil, err
}
io.Copy(part, fieldvalue)
}
writer.Close()
// and sent the reader as the body
return ts.DoRestWithReader(ctx, timeout, method, url, headers.With(RequestHeaders{ContentType: writer.FormDataContentType()}), &buffer)
}
// DoRestWithReader performs a http request where the body is copied from the given io.Reader.
// The caller must ensure the reader is closed.
func (ts Triplestore) DoRestWithMarshal(ctx context.Context, timeout time.Duration, method, url string, headers *RequestHeaders, body any) (*http.Response, error) {
// encode into a buffer
var buffer bytes.Buffer
if err := json.NewEncoder(&buffer).Encode(body); err != nil {
return nil, err
}
return ts.DoRestWithReader(ctx, timeout, method, url, headers.With(RequestHeaders{ContentType: "application/json"}), &buffer)
}
// DoRestWithReader performs a http request where the body is copied from the given io.Reader.
// The caller must ensure the reader is closed.
func (ts Triplestore) DoRestWithReader(ctx context.Context, timeout time.Duration, method string, url string, headers *RequestHeaders, body io.Reader) (*http.Response, error) {
// create the request object
client := &http.Client{
Timeout: timeout,
@ -75,20 +103,22 @@ func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any,
DisableKeepAlives: true,
},
}
req, err := http.NewRequestWithContext(ctx, method, ts.BaseURL+url, reader)
// create the request and authentication
req, err := http.NewRequestWithContext(ctx, method, ts.BaseURL+url, body)
if err != nil {
return nil, err
}
// Setup configuration!
if accept != "" {
req.Header.Set("Accept", accept)
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
req.SetBasicAuth(ts.Config.TS.AdminUsername, ts.Config.TS.AdminPassword)
// add extra headers
if headers != nil && headers.Accept != "" {
req.Header.Set("Accept", headers.Accept)
}
if headers != nil && headers.ContentType != "" {
req.Header.Set("Content-Type", headers.ContentType)
}
// and send it
return client.Do(req)
}
@ -97,7 +127,7 @@ func (ts Triplestore) OpenRaw(ctx context.Context, method, url string, body any,
// This is achieved using a polling strategy.
func (ts Triplestore) Wait(ctx context.Context) error {
return timex.TickUntilFunc(func(time.Time) bool {
res, err := ts.OpenRaw(ctx, "GET", "/rest/repositories", nil, "", "", tsTrivialTimeout)
res, err := ts.DoRest(ctx, tsTrivialTimeout, http.MethodGet, "/rest/repositories", nil)
zerolog.Ctx(ctx).Trace().Err(err).Msg("Triplestore wait")
if err != nil {
return false
@ -110,10 +140,11 @@ func (ts Triplestore) Wait(ctx context.Context) error {
// PurgeUser deletes the specified user from the triplestore.
// When the user does not exist, returns no error.
func (ts Triplestore) PurgeUser(ctx context.Context, user string) error {
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/security/users/"+user, nil, "", "", tsTrivialTimeout)
res, err := ts.DoRest(ctx, tsTrivialTimeout, http.MethodDelete, "/rest/security/users/"+user, nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusNoContent && res.StatusCode != http.StatusNotFound {
return errors.Errorf("Delete returned code %d", res.StatusCode)
}
@ -123,10 +154,11 @@ func (ts Triplestore) PurgeUser(ctx context.Context, user string) error {
// PurgeRepo deletes the specified repo from the triplestore.
// When the repo does not exist, returns no error.
func (ts Triplestore) PurgeRepo(ctx context.Context, repo string) error {
res, err := ts.OpenRaw(ctx, "DELETE", "/rest/repositories/"+repo, nil, "", "", tsTrivialTimeout)
res, err := ts.DoRest(ctx, tsTrivialTimeout, http.MethodDelete, "/rest/repositories/"+repo, nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusNotFound {
return errors.Errorf("Delete returned code %d", res.StatusCode)
}

6
internal/dis/component/triplestore/provision.go
View file

@ -61,7 +61,7 @@ func (ts *Triplestore) CreateRepository(ctx context.Context, name, domain, user,
// do the create!
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/repositories", createRepo.Bytes(), "config", "", tsTrivialTimeout)
res, err := ts.DoRestWithForm(ctx, tsTrivialTimeout, http.MethodPost, "/rest/repositories", nil, "config", &createRepo)
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}
@ -73,7 +73,7 @@ func (ts *Triplestore) CreateRepository(ctx context.Context, name, domain, user,
// create the user and grant them access
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/security/users/"+user, TriplestoreUserPayload{
res, err := ts.DoRestWithMarshal(ctx, tsTrivialTimeout, http.MethodPost, "/rest/security/users/"+user, nil, TriplestoreUserPayload{
Password: password,
AppSettings: TriplestoreUserAppSettings{
DefaultInference: true,
@ -87,7 +87,7 @@ func (ts *Triplestore) CreateRepository(ctx context.Context, name, domain, user,
"READ_REPO_" + name,
"WRITE_REPO_" + name,
},
}, "", "", tsTrivialTimeout)
})
if err != nil {
return errTripleStoreFailedRepository.WithMessageF(err)
}

28
internal/dis/component/triplestore/restore.go Normal file
View file

@ -0,0 +1,28 @@
package triplestore
import (
"context"
"fmt"
"io"
"net/http"
"github.com/pkg/errors"
)
var errTSRestoreWrongStatusCode = errors.New("Triplestore.Restore: Wrong status code")
// RestoreDB snapshots the provided repository into dst
func (ts Triplestore) RestoreDB(ctx context.Context, repo string, reader io.Reader) error {
// submit the form
res, err := ts.DoRestWithReader(ctx, 0, http.MethodPut, "/repositories/"+repo+"/statements", &RequestHeaders{ContentType: nquadsContentType}, reader)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusNoContent {
message, _ := io.ReadAll(res.Body)
return fmt.Errorf("%w: %s", errTSRestoreWrongStatusCode, message)
}
return nil
}

6
internal/dis/component/triplestore/snapshot.go
View file

@ -25,15 +25,17 @@ func (ts *Triplestore) Snapshot(wisski models.Instance, scontext *component.Stag
var errTSBackupWrongStatusCode = errors.New("Triplestore.Backup: Wrong status code")
const nquadsContentType = "text/x-nquads"
// SnapshotDB snapshots the provided repository into dst
func (ts Triplestore) SnapshotDB(ctx context.Context, dst io.Writer, repo string) (int64, error) {
res, err := ts.OpenRaw(ctx, "GET", "/repositories/"+repo+"/statements?infer=false", nil, "", "application/n-quads", 0)
res, err := ts.DoRest(ctx, 0, http.MethodGet, "/repositories/"+repo+"/statements?infer=false", &RequestHeaders{Accept: nquadsContentType})
if err != nil {
return 0, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return 0, errTSBackupWrongStatusCode
}
defer res.Body.Close()
return io.Copy(dst, res.Body)
}

6
internal/dis/component/triplestore/update.go
View file

@ -20,7 +20,7 @@ func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
logging.LogMessage(progress, "Resetting admin user password")
{
res, err := ts.OpenRaw(ctx, "PUT", "/rest/security/users/"+ts.Config.TS.AdminUsername, TriplestoreUserPayload{
res, err := ts.DoRestWithMarshal(ctx, tsTrivialTimeout, http.MethodPut, "/rest/security/users/"+ts.Config.TS.AdminUsername, nil, TriplestoreUserPayload{
Password: ts.Config.TS.AdminPassword,
AppSettings: TriplestoreUserAppSettings{
DefaultInference: true,
@ -30,7 +30,7 @@ func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
ExecuteCount: true,
},
GrantedAuthorities: []string{"ROLE_ADMIN"},
}, "", "", tsTrivialTimeout)
})
if err != nil {
return fmt.Errorf("failed to create triplestore user: %s", err)
}
@ -52,7 +52,7 @@ func (ts Triplestore) Update(ctx context.Context, progress io.Writer) error {
logging.LogMessage(progress, "Enabling Triplestore security")
{
res, err := ts.OpenRaw(ctx, "POST", "/rest/security", true, "", "", tsTrivialTimeout)
res, err := ts.DoRestWithMarshal(ctx, tsTrivialTimeout, http.MethodPost, "/rest/security", nil, true)
if err != nil {
return fmt.Errorf("failed to enable triplestore security: %s", err)
}