From b27871f39a0674c8ef25783206048ebd67dab8a8 Mon Sep 17 00:00:00 2001 From: Tom Wiesing Date: Wed, 2 Nov 2022 13:50:11 +0100 Subject: [PATCH] Add initial support for solr --- cmd/bootstrap.go | 2 +- cmd/monday.go | 2 +- cmd/system_update.go | 2 +- internal/cli/params.go | 2 +- internal/dis/component/solr/solr.env | 1 + internal/dis/component/solr/solr.go | 48 ++++++++++++++ .../component/solr/solr/docker-compose.yml | 18 +++++ internal/dis/distillery.go | 7 ++ internal/dis/init.go | 4 +- pkg/environment/environment.go | 2 +- pkg/environment/native_exec.go | 4 +- pkg/environment/native_fs.go | 66 +++++++++++++------ pkg/environment/native_net.go | 4 +- 13 files changed, 131 insertions(+), 31 deletions(-) create mode 100644 internal/dis/component/solr/solr.env create mode 100644 internal/dis/component/solr/solr.go create mode 100644 internal/dis/component/solr/solr/docker-compose.yml diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 4bb3bec..5713cfc 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -71,7 +71,7 @@ var errBootstrapCreateFile = exit.Error{ func (bs cBootstrap) Run(context wisski_distillery.Context) error { // installation environment is the native environment! // TODO: Should this be configurable? - var env environment.Native + env := new(environment.Native) root := bs.Directory diff --git a/cmd/monday.go b/cmd/monday.go index 32abc98..3107b57 100644 --- a/cmd/monday.go +++ b/cmd/monday.go @@ -30,7 +30,7 @@ func (monday) Description() wisski_distillery.Description { func (monday monday) AfterParse() error { // TODO: Use a generic environment here! - if !fsx.IsFile(environment.Native{}, monday.Positionals.GraphdbZip) { + if !fsx.IsFile(new(environment.Native), monday.Positionals.GraphdbZip) { return errNoGraphDBZip.WithMessageF(monday.Positionals.GraphdbZip) } return nil diff --git a/cmd/system_update.go b/cmd/system_update.go index c526af1..c373897 100644 --- a/cmd/system_update.go +++ b/cmd/system_update.go @@ -46,7 +46,7 @@ var errNoGraphDBZip = exit.Error{ func (s systemupdate) AfterParse() error { // TODO: Use a generic environment here! - if !fsx.IsFile(environment.Native{}, s.Positionals.GraphdbZip) { + if !fsx.IsFile(new(environment.Native), s.Positionals.GraphdbZip) { return errNoGraphDBZip.WithMessageF(s.Positionals.GraphdbZip) } return nil diff --git a/internal/cli/params.go b/internal/cli/params.go index cc9cf6b..929e9b3 100644 --- a/internal/cli/params.go +++ b/internal/cli/params.go @@ -21,7 +21,7 @@ type Params struct { func ParamsFromEnv() (params Params, err error) { // try to read the base directory! - value, err := ReadBaseDirectory(environment.Native{}) // TODO: Are we sure about the native environment here? + value, err := ReadBaseDirectory(new(environment.Native)) // TODO: Are we sure about the native environment here? switch { case environment.IsNotExist(err): params.ConfigPath = bootstrap.BaseDirectoryDefault diff --git a/internal/dis/component/solr/solr.env b/internal/dis/component/solr/solr.env new file mode 100644 index 0000000..131c6ab --- /dev/null +++ b/internal/dis/component/solr/solr.env @@ -0,0 +1 @@ +DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} diff --git a/internal/dis/component/solr/solr.go b/internal/dis/component/solr/solr.go new file mode 100644 index 0000000..ea0a66b --- /dev/null +++ b/internal/dis/component/solr/solr.go @@ -0,0 +1,48 @@ +package solr + +import ( + "context" + "embed" + "path/filepath" + "time" + + "github.com/FAU-CDI/wisski-distillery/internal/dis/component" + "github.com/FAU-CDI/wisski-distillery/pkg/environment" +) + +type Solr struct { + component.Base + + BaseURL string // upstream solr url + + PollContext context.Context // context to abort polling with + PollInterval time.Duration // duration to wait for during wait +} + +func (s *Solr) Path() string { + return filepath.Join(s.Still.Config.DeployRoot, "core", "solr") +} + +func (*Solr) Context(parent component.InstallationContext) component.InstallationContext { + return parent +} + +//go:embed all:solr +//go:embed solr.env +var resources embed.FS + +func (solr *Solr) Stack(env environment.Environment) component.StackWithResources { + return component.MakeStack(solr, env, component.StackWithResources{ + Resources: resources, + ContextPath: "solr", + + EnvPath: "solr.env", + EnvContext: map[string]string{ + "DOCKER_NETWORK_NAME": solr.Config.DockerNetworkName, + }, + + MakeDirs: []string{ + filepath.Join("data"), + }, + }) +} diff --git a/internal/dis/component/solr/solr/docker-compose.yml b/internal/dis/component/solr/solr/docker-compose.yml new file mode 100644 index 0000000..02493e2 --- /dev/null +++ b/internal/dis/component/solr/solr/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.7" + +services: + solr: + image: docker.io/library/solr:8.11-slim + ports: + - "127.0.0.1:8983:8983" + volumes: + - './data/:/var/solr' + labels: + - "eu.wiss-ki.barrel.distillery=${DOCKER_NETWORK_NAME}" + + restart: always + +networks: + default: + name: ${DOCKER_NETWORK_NAME} + external: true \ No newline at end of file diff --git a/internal/dis/distillery.go b/internal/dis/distillery.go index fa4303e..e128d07 100644 --- a/internal/dis/distillery.go +++ b/internal/dis/distillery.go @@ -17,6 +17,7 @@ import ( "github.com/FAU-CDI/wisski-distillery/internal/dis/component/instances/malt" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/meta" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/resolver" + "github.com/FAU-CDI/wisski-distillery/internal/dis/component/solr" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/sql" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/ssh" "github.com/FAU-CDI/wisski-distillery/internal/dis/component/triplestore" @@ -50,6 +51,7 @@ type Distillery struct { type Upstream struct { SQL string Triplestore string + Solr string } // Context returns a new Context belonging to this distillery @@ -114,6 +116,11 @@ func (dis *Distillery) allComponents() []initFunc { sql.PollContext = dis.Context() sql.PollInterval = time.Second }), + manual(func(s *solr.Solr) { + s.BaseURL = dis.Upstream.Solr + s.PollContext = dis.Context() + s.PollInterval = time.Second + }), // instainces auto[*instances.Instances], diff --git a/internal/dis/init.go b/internal/dis/init.go index 945a70b..6049d10 100644 --- a/internal/dis/init.go +++ b/internal/dis/init.go @@ -23,11 +23,12 @@ func NewDistillery(params cli.Params, flags cli.Flags, req cli.Requirements) (di dis = &Distillery{ context: params.Context, Still: component.Still{ - Environment: environment.Native{}, + Environment: new(environment.Native), }, Upstream: Upstream{ SQL: "127.0.0.1:3306", Triplestore: "127.0.0.1:7200", + Solr: "127.0.0.1:8983", }, } @@ -38,6 +39,7 @@ func NewDistillery(params cli.Params, flags cli.Flags, req cli.Requirements) (di if flags.InternalInDocker { dis.Upstream.SQL = "sql:3306" dis.Upstream.Triplestore = "triplestore:7200" + dis.Upstream.Solr = "solr:8983" params.ConfigPath = dis.Still.Environment.GetEnv("CONFIG_PATH") } diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 6ed8a25..ebb7c76 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -55,5 +55,5 @@ type WritableFile interface { } func init() { - var _ Environment = Native{} + var _ Environment = new(Native) } diff --git a/pkg/environment/native_exec.go b/pkg/environment/native_exec.go index 27f19a1..1f6b536 100644 --- a/pkg/environment/native_exec.go +++ b/pkg/environment/native_exec.go @@ -10,7 +10,7 @@ import ( // // If the command executes, it's exit code will be returned. // If the command can not be executed, returns [ExecCommandError]. -func (Native) Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { +func (*Native) Exec(io stream.IOStream, workdir string, exe string, argv ...string) int { // setup the command cmd := exec.Command(exe, argv...) cmd.Dir = workdir @@ -35,7 +35,7 @@ func (Native) Exec(io stream.IOStream, workdir string, exe string, argv ...strin return 0 } -func (n Native) LookPathAbs(file string) (string, error) { +func (n *Native) LookPathAbs(file string) (string, error) { path, err := exec.LookPath(file) if err != nil { return "", err diff --git a/pkg/environment/native_fs.go b/pkg/environment/native_fs.go index dc132ea..382b9a7 100644 --- a/pkg/environment/native_fs.go +++ b/pkg/environment/native_fs.go @@ -4,77 +4,101 @@ import ( "io/fs" "os" "path/filepath" + "sync" + "syscall" "time" ) -type Native struct{} +type Native struct { + ulock sync.Mutex + umask int +} -func (Native) isEnv() {} +func (*Native) isEnv() {} -func (Native) GetEnv(name string) string { +func (n *Native) setMask(umask int) { + n.ulock.Lock() + n.umask = syscall.Umask(umask) +} + +func (n *Native) resetMask() { + syscall.Umask(n.umask) + n.ulock.Unlock() +} + +func (*Native) GetEnv(name string) string { return os.Getenv(name) } -func (Native) Stat(path string) (fs.FileInfo, error) { +func (*Native) Stat(path string) (fs.FileInfo, error) { return os.Stat(path) } -func (Native) Lstat(path string) (fs.FileInfo, error) { +func (*Native) Lstat(path string) (fs.FileInfo, error) { return os.Lstat(path) } -func (Native) Readlink(path string) (string, error) { +func (*Native) Readlink(path string) (string, error) { return os.Readlink(path) } -func (Native) Symlink(oldname, newname string) error { +func (*Native) Symlink(oldname, newname string) error { return os.Symlink(oldname, newname) } -func (Native) ReadDir(name string) ([]fs.DirEntry, error) { +func (*Native) ReadDir(name string) ([]fs.DirEntry, error) { return os.ReadDir(name) } -func (Native) SameFile(f1, f2 fs.FileInfo) bool { +func (*Native) SameFile(f1, f2 fs.FileInfo) bool { return os.SameFile(f1, f2) } -func (Native) WalkDir(path string, f fs.WalkDirFunc) error { +func (*Native) WalkDir(path string, f fs.WalkDirFunc) error { return filepath.WalkDir(path, f) } -func (Native) Executable() (string, error) { +func (*Native) Executable() (string, error) { return os.Executable() // TODO: not sure this works with the remote concepts } -func (Native) Open(path string) (fs.File, error) { +func (*Native) Open(path string) (fs.File, error) { return os.Open(path) } -func (Native) Create(path string, mode fs.FileMode) (WritableFile, error) { +func (n *Native) Create(path string, mode fs.FileMode) (WritableFile, error) { + n.ulock.Lock() + defer n.ulock.Unlock() + return os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) } -func (Native) Chtimes(name string, atime time.Time, mtime time.Time) error { +func (*Native) Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(name, atime, mtime) } -func (Native) Mkdir(path string, mode fs.FileMode) error { - return os.Mkdir(path, mode) +func (n *Native) Mkdir(path string, mode fs.FileMode) error { + n.setMask(0) + defer n.resetMask() + + return os.Mkdir(path, fs.ModeDir|mode) } -func (Native) MkdirAll(path string, mode fs.FileMode) error { - return os.MkdirAll(path, mode) +func (n *Native) MkdirAll(path string, mode fs.FileMode) error { + n.setMask(0) + defer n.resetMask() + + return os.MkdirAll(path, fs.ModeDir|mode) } -func (Native) Remove(path string) error { +func (*Native) Remove(path string) error { return os.Remove(path) } -func (Native) RemoveAll(path string) error { +func (*Native) RemoveAll(path string) error { return os.RemoveAll(path) } -func (Native) Abs(path string) (string, error) { +func (*Native) Abs(path string) (string, error) { return filepath.Abs(path) } diff --git a/pkg/environment/native_net.go b/pkg/environment/native_net.go index 0e6dddf..0443707 100644 --- a/pkg/environment/native_net.go +++ b/pkg/environment/native_net.go @@ -5,11 +5,11 @@ import ( "net" ) -func (Native) Listen(network, address string) (net.Listener, error) { +func (*Native) Listen(network, address string) (net.Listener, error) { return net.Listen(network, address) } -func (Native) DialContext(context context.Context, network, address string) (net.Conn, error) { +func (*Native) DialContext(context context.Context, network, address string) (net.Conn, error) { var d net.Dialer return d.DialContext(context, network, address) }