From 3be5e85ef710abf314575301fd7ac8f825e4de80 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Wed, 21 Sep 2022 16:04:22 +0200 Subject: [PATCH] prefixes: Add 'resolver-blocked.txt' config --- cmd/bootstrap.go | 16 ++++-- cmd/update_prefix_config.go | 4 +- internal/component/control/backup.go | 1 + internal/component/control/control.env | 3 +- internal/component/control/control.go | 1 + .../control/control/docker-compose.yml | 1 + internal/component/instances/wisski_prefix.go | 50 ++++++++++++++----- internal/config/config.go | 4 ++ internal/config/config_template | 5 +- internal/config/template.go | 5 ++ internal/core/bootstrap/resolver-blocked.txt | 16 ++++++ internal/core/core.go | 8 +++ 12 files changed, 95 insertions(+), 19 deletions(-) create mode 100644 internal/core/bootstrap/resolver-blocked.txt diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 60b869a..1dabb92 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -143,7 +143,7 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { core.DefaultOverridesJSON, fs.ModePerm, ); err != nil { - return errBootstrapCreateFile.WithMessageF(err) + return err } context.Println(tpl.AuthorizedKeys) @@ -153,12 +153,22 @@ func (bs bootstrap) Run(context wisski_distillery.Context) error { core.DefaultAuthorizedKeys, fs.ModePerm, ); err != nil { - return errBootstrapCreateFile.WithMessageF(err) + return err + } + + context.Println(tpl.SelfResolverBlockFile) + if err := environment.WriteFile( + env, + tpl.SelfResolverBlockFile, + core.DefaultResolverBlockedTXT, + fs.ModePerm, + ); err != nil { + return err } return nil }, context.IOStream, "Creating additional config files"); err != nil { - return err + return errBootstrapCreateFile.WithMessageF(err) } } diff --git a/cmd/update_prefix_config.go b/cmd/update_prefix_config.go index 454d774..b6d7ab6 100644 --- a/cmd/update_prefix_config.go +++ b/cmd/update_prefix_config.go @@ -53,7 +53,9 @@ func (upc updateprefixconfig) Run(context wisski_distillery.Context) error { // read the prefix config data, err := instance.PrefixConfig() if err != nil { - return err + data = "# error, skipped\n" + context.EPrintln(err) + err = nil } context.IOStream.Printf("%s", data) diff --git a/internal/component/control/backup.go b/internal/component/control/backup.go index 015d02c..4e3b0e7 100644 --- a/internal/component/control/backup.go +++ b/internal/component/control/backup.go @@ -31,6 +31,7 @@ func (control *Control) backupFiles() []string { control.Config.ConfigPath, control.Config.ExecutablePath(), control.Config.SelfOverridesFile, + control.Config.SelfResolverBlockFile, control.Config.GlobalAuthorizedKeysFile, } } diff --git a/internal/component/control/control.env b/internal/component/control/control.env index 939a420..bfd1823 100644 --- a/internal/component/control/control.env +++ b/internal/component/control/control.env @@ -6,4 +6,5 @@ LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} CONFIG_PATH=${CONFIG_PATH} DEPLOY_ROOT=${DEPLOY_ROOT} GLOBAL_AUTHORIZED_KEYS_FILE=${GLOBAL_AUTHORIZED_KEYS_FILE} -SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} \ No newline at end of file +SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} +SELF_RESOLVER_BLOCK_FILE=${SELF_RESOLVER_BLOCK_FILE} \ No newline at end of file diff --git a/internal/component/control/control.go b/internal/component/control/control.go index 7cb8fbd..a1811fe 100644 --- a/internal/component/control/control.go +++ b/internal/component/control/control.go @@ -46,6 +46,7 @@ func (control *Control) Stack(env environment.Environment) component.StackWithRe "GLOBAL_AUTHORIZED_KEYS_FILE": control.Config.GlobalAuthorizedKeysFile, "SELF_OVERRIDES_FILE": control.Config.SelfOverridesFile, + "SELF_RESOLVER_BLOCK_FILE": control.Config.SelfResolverBlockFile, }, TouchFiles: []string{control.ResolverFile}, diff --git a/internal/component/control/control/docker-compose.yml b/internal/component/control/control/docker-compose.yml index b0d028f..1fbd8bd 100644 --- a/internal/component/control/control/docker-compose.yml +++ b/internal/component/control/control/docker-compose.yml @@ -22,6 +22,7 @@ services: - "${DEPLOY_ROOT}:${DEPLOY_ROOT}:ro" - "${GLOBAL_AUTHORIZED_KEYS_FILE}:${GLOBAL_AUTHORIZED_KEYS_FILE}:ro" - "${SELF_OVERRIDES_FILE}:${SELF_OVERRIDES_FILE}:ro" + - "${SELF_RESOLVER_BLOCK_FILE}:${SELF_RESOLVER_BLOCK_FILE}:ro" networks: default: diff --git a/internal/component/instances/wisski_prefix.go b/internal/component/instances/wisski_prefix.go index 8ff7b6d..b1195df 100644 --- a/internal/component/instances/wisski_prefix.go +++ b/internal/component/instances/wisski_prefix.go @@ -1,7 +1,7 @@ package instances import ( - "errors" + "bufio" "io" "path/filepath" "strings" @@ -25,7 +25,7 @@ var listURIPrefixesPHP string // Prefixes returns the prefixes func (wisski *WissKI) Prefixes() (prefixes []string, err error) { // get all the ugly prefixes - err = wisski.ExecPHPScript(stream.FromEnv(), &prefixes, listURIPrefixesPHP, "list_prefixes") + err = wisski.ExecPHPScript(stream.FromNil(), &prefixes, listURIPrefixesPHP, "list_prefixes") if err != nil { return nil, err } @@ -35,29 +35,53 @@ func (wisski *WissKI) Prefixes() (prefixes []string, err error) { return strings.HasPrefix(now, prev) }) + // load the list of blocked prefixes + blocks, err := wisski.instances.BlockedPrefixes() + if err != nil { + return nil, err + } + // filter out blocked prefixes - return slicesx.Filter(prefixes, func(uri string) bool { return !IsNonServedURI(uri) }), nil + return slicesx.Filter(prefixes, func(uri string) bool { return !hasAnyPrefix(uri, blocks) }), nil } -// TODO: Eventually move this into a configuration file. -// But for now this is fine -var blockedURIs = []string{ - "http://erlangen-crm.org/", - "http://www.w3.org/", - "xsd:", +func (instances *Instances) BlockedPrefixes() ([]string, error) { + // open the resolver block file + file, err := instances.Environment.Open(instances.Config.SelfResolverBlockFile) + if err != nil { + return nil, err + } + + var lines []string + + // read all the lines that aren't a comment! + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "#") { + continue + } + lines = append(lines, line) + } + + // check if there was an error + if err := scanner.Err(); err != nil { + return nil, err + } + + // and done! + return lines, nil } -func IsNonServedURI(candidate string) bool { +func hasAnyPrefix(candidate string, prefixes []string) bool { return slicesx.Any( - blockedURIs, + prefixes, func(prefix string) bool { return strings.HasPrefix(candidate, prefix) }, ) } -var errPrefixExecFailed = errors.New("PrefixConfig: Failed to call list_uri_prefixes") - // PrefixConfig returns the prefix config belonging to this instance. func (wisski *WissKI) PrefixConfig() (config string, err error) { // if the user requested to skip the prefix, then don't do anything with it! diff --git a/internal/config/config.go b/internal/config/config.go index 4f7aa6e..66993c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,10 @@ type Config struct { // Do this by adding URLs (without trailing '/'s) into a JSON file SelfOverridesFile string `env:"SELF_OVERRIDES_FILE" default:"" parser:"file"` + // You can block specific prefixes from being picked up by the resolver. + // Do this by adding one prefix per file. + SelfResolverBlockFile string `env:"SELF_RESOLVER_BLOCK_FILE" default:"" parser:"file"` + // The system can support setting up certificate(s) automatically. // It can be enabled by setting an email for certbot certificates. // This email address can be configured here. diff --git a/internal/config/config_template b/internal/config/config_template index c255d85..12b8c67 100644 --- a/internal/config/config_template +++ b/internal/config/config_template @@ -19,6 +19,10 @@ SELF_EXTRA_DOMAINS= # Do this by adding URLs (without trailing '/'s) into a JSON file SELF_OVERRIDES_FILE=${SELF_OVERRIDES_FILE} +# You can block specific prefixes within Triplestore from showing up in the resolver configuration file. +# Do this by adding one prefix per line in this file. +SELF_RESOLVER_BLOCK_FILE=${SELF_RESOLVER_BLOCK_FILE} + # The system can support setting up certificate(s) automatically. # It can be enabled by setting an email for certbot certificates. # This email address can be configured here. @@ -28,7 +32,6 @@ CERTBOT_EMAIL= # Backups older than this will be removed when a new backup is made. MAX_BACKUP_AGE=30 - # Each Drupal instance requires a corresponding system user, database users and databases. # These are also set by the appropriate domain name. # To differentiate them from other users of the system, these names can be prefixed. diff --git a/internal/config/template.go b/internal/config/template.go index 3517149..1c85f1e 100644 --- a/internal/config/template.go +++ b/internal/config/template.go @@ -20,6 +20,7 @@ type Template struct { DeployRoot string `env:"DEPLOY_ROOT"` DefaultDomain string `env:"DEFAULT_DOMAIN"` SelfOverridesFile string `env:"SELF_OVERRIDES_FILE"` + SelfResolverBlockFile string `env:"SELF_RESOLVER_BLOCK_FILE"` AuthorizedKeys string `env:"AUTHORIZED_KEYS_FILE"` TriplestoreAdminUser string `env:"GRAPHDB_ADMIN_USER"` TriplestoreAdminPassword string `env:"GRAPHDB_ADMIN_PASSWORD"` @@ -43,6 +44,10 @@ func (tpl *Template) SetDefaults(env environment.Environment) (err error) { tpl.SelfOverridesFile = filepath.Join(tpl.DeployRoot, core.OverridesJSON) } + if tpl.SelfResolverBlockFile == "" { + tpl.SelfResolverBlockFile = filepath.Join(tpl.DeployRoot, core.ResolverBlockedTXT) + } + if tpl.AuthorizedKeys == "" { tpl.AuthorizedKeys = filepath.Join(tpl.DeployRoot, core.AuthorizedKeys) } diff --git a/internal/core/bootstrap/resolver-blocked.txt b/internal/core/bootstrap/resolver-blocked.txt new file mode 100644 index 0000000..31b4445 --- /dev/null +++ b/internal/core/bootstrap/resolver-blocked.txt @@ -0,0 +1,16 @@ +# This file contains prefixes that should not be picked up by the global resolver. +# They will not appear in the list of prefixes. +# It should be one prefix per line, '#' and '//' as well as blank lines are treated as comments + +# definitely shouldn't be resolved to any WissKI +http://www.w3.org/ +xsd: +urn: + +# generic prefixes by some common adapters. +# you may or may not want these. +http://d-nb.info/gnd/ +http://erlangen-crm.org/ +http://id.gnm.de/ont/ +http://zotero.org/ +https://api.zotero.org/ diff --git a/internal/core/core.go b/internal/core/core.go index b327353..07391ea 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -23,6 +23,14 @@ const OverridesJSON = "overrides.json" //go:embed bootstrap/overrides.json var DefaultOverridesJSON []byte +// ResolverBlockTXT is the name of the resolver blocked prefix file. +// It should be located inside the deployment directory. +const ResolverBlockedTXT = "resolver-blocked.txt" + +// ResolverBlockTXT contains a template for 'resolver-blocked' file +//go:embed bootstrap/resolver-blocked.txt +var DefaultResolverBlockedTXT []byte + // AuthorizedKeys contains the default name for the 'global_authorized_keys' file const AuthorizedKeys = "authorized_keys"