From 7763644ebe6f791bfa3e09c524751bb2e88dbf66 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Mon, 8 Apr 2024 14:43:40 +0200 Subject: [PATCH] Add 'dangerously_use_adapter_prefixes' setting This commit adds a setting to not scan the triplestore for prefixes, but instead use the prefixes listed in adapaters as the only URIs to resolve. --- internal/config/config.yml | 4 ++ internal/config/database.go | 6 +++ .../wisski/ingredient/php/extras/prefixes.go | 43 ++++++++++++++----- .../wisski/ingredient/php/extras/prefixes.php | 30 +++++++++++-- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/internal/config/config.yml b/internal/config/config.yml index 8cac841..938a063 100644 --- a/internal/config/config.yml +++ b/internal/config/config.yml @@ -113,6 +113,10 @@ triplestore: user_prefix: null data_prefix: null + # if desired, turn off regularly scanning for prefixes in the triplestore. + # DANGER: Turning this on will break the global resolver. + dangerously_use_adapter_prefixes: false + # The maximum agefor backups to be kept. # Backups older than this will be removed when a new backup is made. # The default here is 720hours (== 30 days) diff --git a/internal/config/database.go b/internal/config/database.go index 6843162..8a7f503 100644 --- a/internal/config/database.go +++ b/internal/config/database.go @@ -1,5 +1,7 @@ package config +import "github.com/FAU-CDI/wisski-distillery/internal/config/validators" + type DatabaseConfig struct { // Credentials for the admin user. // Is automatically created if it does not exist. @@ -20,4 +22,8 @@ type SQLConfig struct { type TSConfig struct { DatabaseConfig `yaml:",inline" recurse:"true"` + + // DangerouslyUseAdapterPrefixes inidicates if scanning for prefixes should just use prefixes declared in all adapters. + // This may not reflect what is actually in the database. + DangerouslyUseAdapterPrefixes validators.NullableBool `yaml:"dangerously_use_adapter_prefixes" default:"false" validate:"bool"` } diff --git a/internal/wisski/ingredient/php/extras/prefixes.go b/internal/wisski/ingredient/php/extras/prefixes.go index 1c9b9a5..63401ba 100644 --- a/internal/wisski/ingredient/php/extras/prefixes.go +++ b/internal/wisski/ingredient/php/extras/prefixes.go @@ -48,7 +48,7 @@ var listURIPrefixesPHP string // server is an optional server to fetch prefixes from. // server may be nil. func (prefixes *Prefixes) All(ctx context.Context, server *phpx.Server) ([]string, error) { - uris, err := prefixes.triplestore(ctx, server) + uris, err := prefixes.getLivePrefixes(ctx, server) if err != nil { return nil, err } @@ -61,26 +61,49 @@ func (prefixes *Prefixes) All(ctx context.Context, server *phpx.Server) ([]strin return append(uris, uris2...), nil } -func (wisski *Prefixes) triplestore(ctx context.Context, server *phpx.Server) (prefixes []string, err error) { - // get all the ugly prefixes - err = wisski.dependencies.PHP.ExecScript(ctx, server, &prefixes, listURIPrefixesPHP, "list_prefixes") +// getLivePrefixes get the list of prefixes found within the live system +func (prefixes *Prefixes) getLivePrefixes(ctx context.Context, server *phpx.Server) (pfs []string, err error) { + useTS := !(prefixes.Config.TS.DangerouslyUseAdapterPrefixes.Set && prefixes.Config.TS.DangerouslyUseAdapterPrefixes.Value) + + if useTS { + pfs, err = prefixes.getTSPrefixes(ctx, server) + } else { + // danger danger danger: Use the adapter prefixes + pfs, err = prefixes.getAdapterPrefixes(ctx, server) + } + if err != nil { return nil, err } - // filter out sequential prefixes - prefixes = collection.NonSequential(prefixes, func(prev, now string) bool { - return strings.HasPrefix(now, prev) - }) + // sort the prefixes, and remove duplicates + slices.Sort(pfs) + pfs = collection.Deduplicate(pfs) // load the list of blocked prefixes - blocks, err := wisski.blocked() + blocks, err := prefixes.blocked() if err != nil { return nil, err } // filter out blocked prefixes - return collection.Filter(prefixes, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil + return collection.Filter(pfs, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil +} + +func (wisski *Prefixes) getAdapterPrefixes(ctx context.Context, server *phpx.Server) (pfs []string, err error) { + err = wisski.dependencies.PHP.ExecScript(ctx, server, &pfs, listURIPrefixesPHP, "list_adapter_prefixes") + if err != nil { + return nil, err + } + return pfs, nil +} + +func (wisski *Prefixes) getTSPrefixes(ctx context.Context, server *phpx.Server) (pfs []string, err error) { + err = wisski.dependencies.PHP.ExecScript(ctx, server, &pfs, listURIPrefixesPHP, "list_triplestore_prefixes") + if err != nil { + return nil, err + } + return pfs, nil } func (prefixes *Prefixes) blocked() ([]string, error) { diff --git a/internal/wisski/ingredient/php/extras/prefixes.php b/internal/wisski/ingredient/php/extras/prefixes.php index fd4b5e7..ba233f7 100644 --- a/internal/wisski/ingredient/php/extras/prefixes.php +++ b/internal/wisski/ingredient/php/extras/prefixes.php @@ -1,16 +1,16 @@ getStorage('wisski_salz_adapter'); foreach ($storage->loadMultiple() as $adapter) { // load all the prefixes from the triplestore $engine = $adapter->getEngine(); - getTriplestorePrefixes($adapter->getEngine(), $prefixes); + get_prefixes_from_engine($adapter->getEngine(), $prefixes); // read the configuration to check if we have a default graph $conf = $engine->getConfiguration(); @@ -22,7 +22,29 @@ function list_prefixes() { return $prefixes; } -function getTriplestorePrefixes($engine, &$prefixes) { + +/** + * list_adapter_prefixes returns the prefixes of all adapters. + * Prefixes are not filtered, and may contain duplicates. + */ +function list_adapter_prefixes() { + $prefixes = []; + $storage = \Drupal::entityTypeManager()->getStorage('wisski_salz_adapter'); + foreach ($storage->loadMultiple() as $adapter) { + // load all the prefixes from the triplestore + $engine = $adapter->getEngine(); + + // read the configuration to check if we have a default graph + $conf = $engine->getConfiguration(); + if(!array_key_exists('default_graph', $conf)) { + continue; + } + $prefixes[] = $conf['default_graph']; + } + return $prefixes; +} + +function get_prefixes_from_engine($engine, &$prefixes) { // some adapters don't support a query method! if (!method_exists($engine, 'directQuery')) return NULL;