Move to yaml-based configuration

This commit updates the configuration to be yaml-based and updates the
configuration to read in a yaml file.
This commit is contained in:
Tom Wiesing 2023-02-12 18:13:52 +01:00
parent 568c005d15
commit 945329a080
No known key found for this signature in database
70 changed files with 1150 additions and 350 deletions

View file

@ -1,117 +0,0 @@
// Package envreader
package envreader
import (
"bufio"
"io"
"strings"
)
// Scanner is a scanner for environment files.
// To create a new scanner use [NewScanner].
//
// It scans through a reader and reads environment variables from it.
// Reads may be internally buffered.
//
// An environment variable is of the form:
// KEY=VALUE
// on a separate line.
// Keys and values are case-sensitive and may contain anything except for newline characters.
// Spaces around key and value are trimmed using [strings.TrimSpace].
// Keys may not contain an '='.
// Lines not containing a '=' (e.g. blank lines) and those starting with '#' and '//' are ignored.
//
// To advance the scanner to the next key, value pair use [Scan].
// To get the current (key, value) pair, use [Data].
//
// A typical use-case of a scanner is as follows:
//
// scanner := NewScanner(r)
// for scanner.Scan() {
// // process any data ....
// fmt.Println(scanner.Data())
// }
// if err := scanner.Err(); err != nil {
// // handle errors
// }
//
// For the common use case of reading a set of distinct keys from a file see [ReadAll].
type Scanner struct {
s *bufio.Scanner
// current key and value
key string
value string
}
// NewScanner creates a new scanner from the underlying Reader
func NewScanner(r io.Reader) *Scanner {
return &Scanner{
s: bufio.NewScanner(r),
}
}
// Scanner advances the scanner until the next KEY=VALUE pair.
//
// If there are no more values left (e.g. the underlying reader returned io.EOF)
// or when an unexpected error occured, returns false.
//
// A caller should always check Err() to see if there was an error.
func (scanner *Scanner) Scan() bool {
var found bool
for scanner.s.Scan() {
// check that we don't have an empty or comment only line
tokens := strings.TrimSpace(scanner.s.Text())
if len(tokens) == 0 || tokens[0] == '#' || strings.HasPrefix(tokens, "//") {
continue
}
// check that we have a 'key=value' pair
scanner.key, scanner.value, found = strings.Cut(tokens, "=")
if !found {
continue
}
// got a key = value
scanner.key = strings.TrimSpace(scanner.key)
scanner.value = strings.TrimSpace(scanner.value)
return true
}
// nothing found
scanner.key = ""
scanner.value = ""
return false
}
// Data reads the current value from the scanner.
// When Scan() has not been called, or returned false, returns two empty strings.
func (scanner Scanner) Data() (key, value string) {
return scanner.key, scanner.value
}
// Err returns any error that occured on the underlying read.
//
// When no error occured, or the underlying read is io.EOF, returns nil.
func (scanner Scanner) Err() error {
return scanner.s.Err()
}
// ReadAll creates a new [Scanner], and then reads all key/value pairs from r.
// If a key occurs more than once, only the last value is set in the returned map.
func ReadAll(r io.Reader) (values map[string]string, err error) {
scanner := NewScanner(r)
// read and store all values
values = make(map[string]string)
for scanner.Scan() {
key, value := scanner.Data()
values[key] = value
}
// check if there was an error!
if err := scanner.Err(); err != nil {
return nil, err
}
return values, nil
}

View file

@ -1,42 +0,0 @@
// Package envreader
package envreader
import (
"fmt"
"strings"
)
func ExampleNewScanner() {
scanner := NewScanner(strings.NewReader(`
lines without an equal sign are ignored
// this line is a comment, even with an = sign
KEY=VALUE
# this is also a comment =
spaces in keys = spaces in values
multiple=equal=signs
CaSe = SenSitiVe
empty value=
=empty key
`))
for scanner.Scan() {
key, value := scanner.Data()
fmt.Printf("%q %q\n", key, value)
}
if err := scanner.Err(); err != nil {
fmt.Println(scanner.Err())
} else {
fmt.Println("no error")
}
// Output: "KEY" "VALUE"
// "spaces in keys" "spaces in values"
// "multiple" "equal=signs"
// "CaSe" "SenSitiVe"
// "empty value" ""
// "" "empty key"
// no error
}

View file

@ -1,65 +0,0 @@
package stringparser
import (
"reflect"
"strings"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/pkg/errors"
)
var errUnknownParser = errors.New("unknown parser")
// Parse parses the provided value with the parser.
func Parse(env environment.Environment, name, value string, vField reflect.Value) error {
// use the validator
parser, ok := knownParsers[strings.ToLower(name)]
if parser == nil || !ok {
return errUnknownParser
}
// get the parsed value
checked, err := parser(env, value)
if err != nil {
return err
}
// set the value of the field
var errSet interface{}
func() {
defer func() {
errSet = recover()
}()
vField.Set(reflect.ValueOf(checked))
}()
// capture any error
if errSet != nil {
return errors.Errorf("set returned %v", name, errSet)
}
return nil
}
// knownParsers holds the known parsers
var knownParsers map[string]Parser[any] = map[string]Parser[any]{
"abspath": asGenericParser(ParseAbspath),
"domain": asGenericParser(ParseValidDomain),
"domains": asGenericParser(ParseValidDomains),
"duration": asGenericParser(ParseDuration),
"number": asGenericParser(ParseNumber),
"port": asGenericParser(ParsePort),
"https_url": asGenericParser(ParseHttpsURL),
"slug": asGenericParser(ParseSlug),
"file": asGenericParser(ParseFile),
"email": asGenericParser(ParseEmail),
"nonempty": asGenericParser(ParseNonEmpty),
}
func asGenericParser[T any](parser Parser[T]) Parser[any] {
return func(env environment.Environment, s string) (value any, err error) {
value, err = parser(env, s)
return
}
}

View file

@ -1,124 +0,0 @@
// Package stringparser provides Parser
package stringparser
import (
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/FAU-CDI/wisski-distillery/pkg/environment"
"github.com/FAU-CDI/wisski-distillery/pkg/fsx"
"github.com/pkg/errors"
)
// Parser is used to read a value from a string and turn it into a golang value.
// It is simultaniously used to validate particular setting.
//
// Parsers can be found in this package as functions called Parse*.
// They are refered to by their name, e.g. ParseNonempty can be refered to by the name 'Nonempty'.
// See [Parse].
type Parser[T any] func(env environment.Environment, s string) (T, error)
// ParseAbspath checks that s is an absolute path and returns it as-is
func ParseAbspath(env environment.Environment, s string) (string, error) {
if !fsx.IsDirectory(env, s) {
return "", errors.Errorf("%q does not exist or is not a directory", s)
}
return s, nil
}
// ParseFile checks that s is a valid file and returns it as-is
func ParseFile(env environment.Environment, s string) (string, error) {
if !fsx.IsFile(env, s) {
return "", errors.Errorf("%q does not exist or is not a regular file", s)
}
return s, nil
}
var errEmptyString = errors.New("value is empty")
// ParseNonEmpty checks that s is a non-empty string and returns it as-is
func ParseNonEmpty(env environment.Environment, s string) (string, error) {
if s == "" {
return "", errEmptyString
}
return s, nil
}
var regexpDomain = regexp.MustCompile(`^([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseValidDomain checks that s is a valid domain and returns it in lowercase
func ParseValidDomain(env environment.Environment, s string) (string, error) {
if !regexpDomain.MatchString(s) {
return "", errors.Errorf("%q is not a valid domain", s)
}
return strings.ToLower(s), nil
}
// ParseValidDomains checks that s is a comma-seperated list of valid domains and returns them in lower case
func ParseValidDomains(env environment.Environment, s string) ([]string, error) {
if len(s) == 0 {
return []string{}, nil
}
domains := strings.Split(strings.ToLower(s), ",")
for _, d := range domains {
if !regexpDomain.MatchString(d) {
return nil, errors.Errorf("%q is not a valid domain", d)
}
}
return domains, nil
}
// ParseNumber parses s as a decimal integer
func ParseNumber(env environment.Environment, s string) (int, error) {
value, err := strconv.ParseInt(s, 10, 64)
return int(value), err
}
// ParsePort parses s as a port
func ParsePort(env environment.Environment, s string) (uint16, error) {
value, err := strconv.ParseUint(s, 10, 16)
return uint16(value), err
}
// ParseHttpsURL parses a string into a url that starts with 'https://'
func ParseHttpsURL(env environment.Environment, s string) (*url.URL, error) {
url, err := url.Parse(s)
if err != nil {
return nil, errors.Wrapf(err, "%q is not a valid URL", s)
}
if url.Scheme != "https" {
return nil, errors.Errorf("%q is not a valid https URL (%q)", s, url.Scheme)
}
return url, nil
}
var regexpEmail = regexp.MustCompile(`^([-a-zA-Z0-9]+)\@([a-zA-Z0-9][-a-zA-Z0-9]*\.)*[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseEmail checks that s represents an email, and then returns it as is.
func ParseEmail(env environment.Environment, s string) (string, error) {
if s == "" { // no email provided
return "", nil
}
if !regexpEmail.MatchString(s) {
return "", errors.Errorf("%q is not a valid email", s)
}
return s, nil
}
var regexpSlug = regexp.MustCompile(`^[a-zA-Z0-9][-a-zA-Z0-9]*$`) // TODO: Make this regexp nicer!
// ParseSlug parses s as a slug and returns it as is.
func ParseSlug(env environment.Environment, s string) (string, error) {
if !regexpSlug.MatchString(s) {
return "", errors.Errorf("%q is not a valid slug", s)
}
return s, nil
}
// ParseDuration parses a time.Duration
func ParseDuration(env environment.Environment, s string) (time.Duration, error) {
return time.ParseDuration(s)
}

View file

@ -0,0 +1,88 @@
package validator
import (
"errors"
"fmt"
"reflect"
)
const (
validateTag = "validate"
recurseTag = "recurse"
dfltTag = "default"
)
// Validate validates an object of type T, setting defaults where appropriate.
//
// T must be a struct type, when this is not the case, returns ErrNotAStruct.
// validators should contain a set of validators.
//
// Validate iterates over the fields and tags of those fields as follows:
// - If the 'validate' tag is not the empty string, read the appropriate validator from the map, and call the function.
// If the element in the validators map does not exist, returns an error that unwraps to type UnknownValidator.
// If the element in the validators map is not a validator, returns an error that unwraps to type NotAValidator.
// If the type of validator function does not match the field type, returns an error that unwraps to type IncompatibleValidator.
// - If the 'recurse' tag is not the empty string, recurse into the struct type by calling Validate on it.
// If the annotated field is not a struct, return an error.
//
// Any error is wrapped in a FieldError, indicating the field they occured in.
// Recursive validate calls may result in FieldError wraps.
// For a description of struct tags, see [reflect.StructTag].
func Validate[T any](data *T, validators map[string]any) error {
return validate(reflect.ValueOf(data).Elem(), validators)
}
// FieldError wraps an error to indicate which field it occured in.
type FieldError struct {
Field string
Err error
}
func (fe FieldError) Error() string {
return fmt.Sprintf("field %q: %s", fe.Field, fe.Err)
}
func (fe FieldError) Unwrap() error {
return fe.Err
}
var ErrNotAStruct = errors.New("validate called on non-struct type")
func validate(datum reflect.Value, validators Collection) error {
// make sure that we have a struct type
typ := datum.Type()
if typ.Kind() != reflect.Struct {
return ErrNotAStruct
}
fieldC := typ.NumField()
for i := 0; i < fieldC; i++ {
field := typ.Field(i)
// if the recurse tag is set, do the recursion!
if field.Tag.Get(recurseTag) != "" {
if err := validate(datum.FieldByName(field.Name), validators); err != nil {
return FieldError{Field: field.Name, Err: err}
}
continue
}
// check if there is a validator associated with this tag
// and if not, skip it!
validator := field.Tag.Get(validateTag)
if validator == "" {
continue
}
// call the actual validator
if err := validators.Call(
validator,
datum.FieldByName(field.Name),
field.Tag.Get(dfltTag),
); err != nil {
return FieldError{Field: field.Name, Err: err}
}
}
return nil
}

View file

@ -0,0 +1,123 @@
package validator
import (
"errors"
"fmt"
"strconv"
)
func ExampleValidate() {
var value struct {
Number int `validate:"positive" default:"234"`
String string `validate:"nonempty" default:"stuff"`
Recursive struct {
Number int `validate:"positive" default:"45"`
String string `validate:"nonempty" default:"more"`
} `recurse:"true"`
}
collection := make(Collection, 2)
Add(collection, "positive", func(value *int, dflt string) error {
if *value == 0 {
i, err := strconv.ParseInt(dflt, 10, 64)
if err != nil {
return err
}
*value = int(i)
return nil
}
if *value < 0 {
return errors.New("not positive")
}
return nil
})
Add(collection, "nonempty", func(value *string, dflt string) error {
if *value == "" {
*value = dflt
}
if *value == "" {
return errors.New("empty string")
}
return nil
})
err := Validate(&value, collection)
fmt.Printf("%v\n", value)
fmt.Println(err)
// Output: {234 stuff {45 more}}
// <nil>
}
func ExampleValidate_fail() {
var value struct {
Number int `validate:"positive" default:"12"`
String string `validate:"nonempty" default:"stuff"`
Recursive struct {
Number int `validate:"positive" default:"12"`
String string `validate:"nonempty"`
} `recurse:"true"`
}
collection := make(Collection, 2)
Add(collection, "positive", func(value *int, dflt string) error {
if *value == 0 {
i, err := strconv.ParseInt(dflt, 10, 64)
if err != nil {
return err
}
*value = int(i)
return nil
}
if *value < 0 {
return errors.New("not positive")
}
return nil
})
Add(collection, "nonempty", func(value *string, dflt string) error {
if *value == "" {
*value = dflt
}
if *value == "" {
return errors.New("empty string")
}
return nil
})
err := Validate(&value, collection)
fmt.Printf("%v\n", value)
fmt.Println(err)
// Output: {12 stuff {12 }}
// field "Recursive": field "String": empty string
}
func ExampleValidate_notastruct() {
var value int
err := Validate(&value, nil)
fmt.Println(err)
}
func ExampleValidate_notavalidator() {
var value struct {
Field int `validate:"generic"`
}
collection := make(Collection, 2)
collection["generic"] = func(x, y int) error {
panic("never reached")
}
err := Validate(&value, collection)
fmt.Println(err)
// Output: field "Field": entry "generic" in validators is not a valiator
}
func ExampleValidate_invalid() {
var value struct {
Field int `validate:"string"`
}
collection := make(Collection, 2)
collection["string"] = func(value *string, dflt string) error {
panic("never reached")
}
err := Validate(&value, collection)
fmt.Println(err)
// Output: field "Field": validator "string": got type string, expected type int
}

131
pkg/validator/vmap.go Normal file
View file

@ -0,0 +1,131 @@
package validator
import (
"fmt"
"reflect"
"strings"
"github.com/tkw1536/goprogram/lib/reflectx"
)
// Collection represents a set of validators.
// The zero value is not ready to use; it should be created using make().
//
// A validator is a non-nil function with signature func(value *F, dflt string) error.
// Here F is the type of a value of a field.
// The value is the initialized value to be validated.
// The validator may perform abitrary normalization on the value.
// dflt is the default value (read from the 'default' tag).
// error should be an appropriate error that occured.
//
// A validator function is applied by calling it.
type Collection map[string]any
// Add adds a Validator to the provided collection of validators.
// Any previously validator of the same name is overwritten.
func Add[F any](coll Collection, name string, validator func(value *F, dflt string) error) {
coll[name] = validator
}
// AddSlice adds a Validator to the provided collection of validators that validates a slice of the given type. The default is seperated by seperator.
func AddSlice[F any](coll Collection, name string, sep string, validator func(value *F, dflt string) error) {
Add(coll, name, func(value *[]F, dflt string) error {
// some value is set, so we do not need to set the default!
if *value != nil {
for i := range *value {
if err := validator(&(*value)[i], ""); err != nil {
return err
}
}
}
// no default provided => set if to an empty slice
if dflt == "" {
*value = make([]F, 0)
return nil
}
// some default provided => iterate over the underlying validator
dflts := strings.Split(dflt, sep)
*value = make([]F, len(dflts))
for i := range *value {
if err := validator(&(*value)[i], dflts[i]); err != nil {
return err
}
}
return nil
})
}
var (
errTyp = reflectx.TypeOf[error]()
strTyp = reflectx.TypeOf[string]()
)
// UnknownValidator is an error returned from Validate if a validator does not exist
type UnknownValidator string
func (uv UnknownValidator) Error() string {
return fmt.Sprintf("unknown validator %q", string(uv))
}
// NotAValidator is an error returned from Validate if an entry in the validators map is not a validator
type NotAValidator string
func (nv NotAValidator) Error() string {
return fmt.Sprintf("entry %q in validators is not a valiator", string(nv))
}
// IncompatibleValidator is returned when a validator in the validators map is incompatible
type IncompatibleValidator struct {
Validator string
GotType reflect.Type
ExpectedType reflect.Type
}
func (iv IncompatibleValidator) Error() string {
return fmt.Sprintf("validator %q: got type %s, expected type %s", iv.Validator, iv.GotType, iv.ExpectedType)
}
// Call calls the validator with the given name, on the given value, and with the provided default.
// See documentation of [Validate] for details.
func (coll Collection) Call(name string, field reflect.Value, dflt string) error {
validator, ok := coll[name]
if !ok {
return UnknownValidator(name)
}
// get the type of the validator
vFunc := reflect.ValueOf(validator)
vTyp := vFunc.Type()
// ensure that vTyp is of type func(*F,string) error
// where T is the type of the field
//
// - the first if assumes checks for some type F
// - the second if checks if the F is the right one
if validator == nil || vTyp.Kind() != reflect.Func || // func
vTyp.NumIn() != 2 || vTyp.In(0).Kind() != reflect.Pointer || vTyp.In(1) != strTyp || // (*F,string)
vTyp.NumOut() != 1 || vTyp.Out(0) != errTyp { // error
return NotAValidator(name)
}
if vTyp.In(0).Elem() != field.Type() { // the correct *F
return IncompatibleValidator{
Validator: name,
GotType: vTyp.In(0).Elem(),
ExpectedType: field.Type(),
}
}
// call the validator function, and return an error
results := vFunc.Call([]reflect.Value{field.Addr(), reflect.ValueOf(dflt)})
// turn the result into an error
// NOTE: We can't just .(error) here because that panic()s on err == nil
err := results[0].Interface()
if err, ok := err.(error); ok {
return err
}
return nil
}