rnsrk/wisski-cloud-distillery
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

embed: Begin refactor to use clearer paths

Browse source
This commit is contained in:
Tom Wiesing 2022-09-11 12:47:00 +02:00
parent e75dc29de1
commit e1ee569629
No known key found for this signature in database
16 changed files with 431 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

31
internal/fsx/open.go Normal file
View file

@ -0,0 +1,31 @@
package fsx
import "io/fs"
// OpenFS opens the named file in filesystem.
// If opening the file results in an error, returns [ErrFile].
func OpenFS(name string, fsys fs.FS) fs.File {
file, err := fsys.Open(name)
if err != nil {
return ErrFile{Err: err}
}
return file
}
// ErrFile implements a no-op [fs.File].
//
// Every operation will return an underlying error
type ErrFile struct {
Err error
}
func (err ErrFile) Stat() (fs.FileInfo, error) {
return nil, err.Err
}
func (err ErrFile) Read([]byte) (int, error) {
return 0, err.Err
}
func (err ErrFile) Close() error {
return err.Err
}

24
internal/stack/installable.go
View file

@ -7,6 +7,7 @@ import (
"github.com/FAU-CDI/wisski-distillery/embed"
"github.com/FAU-CDI/wisski-distillery/internal/fsx"
"github.com/FAU-CDI/wisski-distillery/internal/unpack"
"github.com/pkg/errors"
"github.com/tkw1536/goprogram/stream"
)
@ -16,10 +17,14 @@ import (
type Installable struct {
Stack
ContextResource string // Path to the resource containing 'docker compose' context
EnvFileResource string // Path to the resource containing dynamically generated env file
EnvFileContext map[string]string // Context of variables to replace in the env file
// Installable enabled installing several resources from a (potentially embedded) filesystem.
//
// The Resources holds these, with appropriate resources specified below.
// These all refer to paths within the Resource filesystem.
Resources fs.FS
ContextPath string // the 'docker compose' stack context, containing e.g. 'docker-compose.yml'.
EnvPath string // the '.env' template, will be installed using [unpack.InstallTemplate].
EnvContext map[string]string // context when instantiating the '.env' template
CopyContextFiles []string // Files to copy from the installation context
@ -40,7 +45,7 @@ func (is Installable) Install(io stream.IOStream, context InstallationContext) e
// setup the base files
if err := embed.InstallResource(
is.Dir,
is.ContextResource,
is.ContextPath,
func(dst, src string) {
io.Printf("[install] %s\n", dst)
},
@ -50,12 +55,13 @@ func (is Installable) Install(io stream.IOStream, context InstallationContext) e
// configure .env
envDest := filepath.Join(is.Dir, ".env")
if is.EnvFileResource != "" && is.EnvFileContext != nil {
if is.EnvPath != "" && is.EnvContext != nil {
io.Printf("[config] %s\n", envDest)
if err := embed.InstallTemplate(
if err := unpack.InstallTemplate(
envDest,
is.EnvFileResource,
is.EnvFileContext,
is.EnvContext,
is.EnvPath,
is.Resources,
); err != nil {
return err
}

82
internal/unpack/dir.go Normal file
View file

@ -0,0 +1,82 @@
package unpack
import (
"io/fs"
"os"
"path/filepath"
"github.com/pkg/errors"
)
// InstallDir installs the directory at src within fsys to dst.
//
// onInstallFile is called for each file or directory being installed.
//
// If the destination path does not exist, it is created using [os.MakeDirs]
// The directory is installed recursively.
func InstallDir(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the source file
srcFile, err := fsys.Open(src)
if err != nil {
return err
}
// stat it!
srcInfo, err := srcFile.Stat()
if err != nil {
return err
}
// make sure it's a file!
if !srcInfo.IsDir() {
return errExpectedDirectoryButGotFile
}
// call the hook (if any)
if onInstallFile != nil {
onInstallFile(dst, src)
}
// do the installation of the directory.
// the type cast should be safe.
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
func installDir(dst string, srcInfo fs.FileInfo, srcFile fs.ReadDirFile, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// create the destination
dstStat, dstErr := os.Stat(dst)
switch {
case os.IsNotExist(dstErr):
if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
return errors.Wrapf(err, "Error creating destination directory %s", dst)
}
case dstErr != nil:
return errors.Wrapf(dstErr, "Error calling stat on destination %s", dst)
case !dstStat.IsDir():
return errors.Wrapf(errExpectedDirectoryButGotFile, "Error opening destination %s", dst)
}
// NOTE(twiesing): We don't use fs.Walk here.
// If we did, we'd have to reconstruct relative paths.
// That would be very ugly!
// read the directory
entries, err := srcFile.ReadDir(-1)
if err != nil {
return errors.Wrapf(err, "Error reading source directory %s", srcFile)
}
// iterate over all the children
for _, entry := range entries {
if err := InstallResource(
filepath.Join(dst, entry.Name()),
filepath.Join(src, entry.Name()),
fsys,
onInstallFile,
); err != nil {
return err
}
}
return nil
}

76
internal/unpack/file.go Normal file
View file

@ -0,0 +1,76 @@
package unpack
import (
"io"
"io/fs"
"os"
"github.com/pkg/errors"
)
// InstallFile installs the file from src into dst.
//
// If the destination path does not exist, it is created.
func InstallFile(dst string, src fs.File) error {
// stat it!
srcInfo, err := src.Stat()
if err != nil {
return err
}
// if this is a directory, something went wrong!
if srcInfo.IsDir() {
return errExpectedFileButGotDirectory
}
// and store it there!
return installFile(dst, srcInfo, src)
}
func installFile(dst string, srcInfo fs.FileInfo, src fs.File) error {
// create the file using the right mode!
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
if err != nil {
return err
}
defer file.Close()
// copy over the content!
_, err = io.Copy(file, src)
return errors.Wrapf(err, "Error writing to destination %s", dst)
}
// InstallTemplate unpacks the resource located at src in fsys, then processes it as a template, and eventually writes it to dst.
// Any existing file is truncated and overwritten.
//
// See [WriteTemplate] for possible errors.
func InstallTemplate(dst string, context map[string]string, src string, fsys fs.FS) error {
// open the srcFile
srcFile, err := fsys.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
// stat it
srcInfo, err := srcFile.Stat()
if err != nil {
return err
}
// check if it is a directory
if srcInfo.IsDir() {
return errExpectedFileButGotDirectory
}
// open the destination file
file, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
if err != nil {
return err
}
defer file.Close()
// write the file!
return WriteTemplate(file, context, srcFile)
}

186
internal/unpack/template.go
View file

@ -3,119 +3,140 @@ package unpack
import (
"bufio"
"bytes"
"fmt"
"io"
"io/fs"
"strings"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
var errExpectedFileButGotDirectory = errors.New("Expected a file, but got a directory")
// UnpackTemplate unpacks the given file template and template
func UnpackTemplate(context map[string]string, src fs.File) ([]byte, fs.FileMode, error) {
// stat the source file to install
srcStat, srcErr := src.Stat()
if srcErr != nil {
return nil, 0, errors.Wrapf(srcErr, "Error calling stat on source")
}
// should not be a directory
if srcStat.IsDir() {
return nil, 0, errors.Wrapf(errExpectedFileButGotDirectory, "Error calling stat on source %s", srcStat.Name())
}
// read all the bytes into a buffer
var buffer bytes.Buffer
WriteTemplate(&buffer, context, src)
return buffer.Bytes(), srcStat.Mode(), nil
}
type templateMode int
// ts represents state of the template parser
type ts int
const (
templateModeNormal templateMode = iota // normal mode
templateModeDollar // saw '$'
templateModeOpen // saw '${'
tsGobble ts = iota // gobble into dst
tsSawDollar // saw a '$'
tsGobbleVar // gobble into var
)
// MissingTemplateKeyError indicates [WriteTemplate] found missing keys in the context
type MissingTemplateKeyError struct {
Keys []string
}
func (mtke MissingTemplateKeyError) Error() string {
return fmt.Sprintf("missing template keys from context: %v", mtke.Keys)
}
// UnusuedTemplateKeyError indicates [WriteTemplate] found unusued keys in the context
type UnusuedTemplateKeyError struct {
Keys []string
}
func (utke UnusuedTemplateKeyError) Error() string {
return fmt.Sprintf("unused template keys from context: %v", utke.Keys)
}
// WriteTemplate writes the template defined by src with the given context into reader.
//
// To run the template, variables of the form ${NAME} are replaced with their corresponding value from the context.
//
// Extra or missing variables from the context are an error.
// If an underlying read or write fails, it is returned as is.
// Missing template keys return a [MissingTemplateKeyError], but are replaced with the empty string.
// Unused template keys return a [UnusuedTemplateKeyError], but are replaced with the empty string.
//
// Reader / Writer errors are always returned first; next missing template keys, and finally unused template keys.
func WriteTemplate(dst io.Writer, context map[string]string, src io.Reader) error {
// keep track of context keys that have not been used
unuusedContext := make(map[string]struct{}, len(context))
// We keep track of contect keys that have not been used.
//
// We first fill the map with all the keys from the context.
// Then when we use a key, we delete it from the map.
// If there are any keys left at the end of the replacement, that is an error.
unusedKeys := make(map[string]struct{}, len(context))
for key := range context {
unuusedContext[key] = struct{}{}
unusedKeys[key] = struct{}{}
}
reader := bufio.NewReader(src) // a new reader
var missingKeyErr error // error for missing keys
var builder strings.Builder // holding variable names
mode := templateModeNormal // the current mode of the reader
// When we encounter a missing key, put it into this map.
// This is so that we can build an error message below.
missingKeys := make(map[string]struct{}, 0)
// We use a new bufio reader to read data from the input.
// This is a cheap trick to get a ReadRune() method.
reader := bufio.NewReader(src)
//
// MAIN PARSING LOOP
//
// start out in gobble mode!
mode := tsGobble
// keep track of variable names
var varB strings.Builder
parseloop:
for {
r, _, err := reader.ReadRune()
switch {
case err == io.EOF:
/* finished the source, see below */
// finished parsing the source
break parseloop
case err != nil:
/* something went wrong */
// the reader broke
return err
case mode == templateModeNormal && r == '$':
// saw a '$' in normal mode
// => switch to the dollar mode
mode = templateModeDollar
case mode == templateModeNormal:
// saw anything else
// => just pass it through
case mode == tsGobble && r == '$':
// saw a '$' in gobble mode
mode = tsSawDollar
case mode == tsGobble:
// normal gobbleing
// => pass it through
if _, err := dst.Write([]byte{byte(r)}); err != nil {
return err
}
case mode == templateModeDollar && r == '{':
case mode == tsSawDollar && r == '{':
// saw '{', following the '$'
// => read everything else into the buffer
mode = templateModeOpen
case mode == templateModeDollar && r == '$':
mode = tsGobbleVar
case mode == tsSawDollar && r == '$':
// saw a '$' following the '$'
// => write the first '$', and handle the case $${stuff}
if _, err := dst.Write([]byte("$")); err != nil {
return err
}
case mode == templateModeDollar:
case mode == tsSawDollar:
// saw anything else following the '$'
// => write both back and switch back to normal mode
// => write both back and switch back to gobble mode
if _, err := dst.Write([]byte{byte('$'), byte(r)}); err != nil {
return err
}
mode = templateModeNormal
mode = tsGobble
case mode == templateModeOpen && r != '}':
case mode == tsGobbleVar && r != '}':
// saw anything except for closing bracket
// => keep it in the buffer
if _, err := builder.WriteRune(r); err != nil {
if _, err := varB.WriteRune(r); err != nil {
return err
}
case mode == templateModeOpen:
// saw a closing '}' inside the open mode
case mode == tsGobbleVar:
// saw a closing '}' inside tsGobbleVar mode
// => use the variable
name := builder.String()
delete(unuusedContext, name) // mark the variable as used
name := varB.String()
// get the variable from the context
value, ok := context[name]
if missingKeyErr != nil && !ok {
missingKeyErr = errors.Errorf("key %s missing in context", name)
delete(unusedKeys, name) // mark the variable as used!
if !ok {
// store unusued variables
missingKeys[name] = struct{}{}
value = ""
}
// write the replacement into the string
@ -124,48 +145,51 @@ parseloop:
}
// reset the builder and go back into normal mode
builder.Reset()
mode = templateModeNormal
default:
panic("never reached")
varB.Reset()
mode = tsGobble
}
}
// cleanup at end of input
//
// CLEANUP UNUSUED INPUT
//
switch mode {
case templateModeNormal:
// => everything is fine
case templateModeDollar:
case tsSawDollar:
// we had a '$', but no '{'
// => write the trailing '$' into dest
if _, err := dst.Write([]byte("$")); err != nil {
return err
}
case templateModeOpen:
case tsGobbleVar:
// we had a "${", followed by somthing unclosed
// => write everything back into the dst
if _, err := dst.Write([]byte("${")); err != nil {
return err
}
if _, err := io.WriteString(dst, builder.String()); err != nil {
if _, err := io.WriteString(dst, varB.String()); err != nil {
return err
}
default:
panic("never reached")
}
// check if there was a missing key!
if missingKeyErr != nil {
return missingKeyErr
}
// check if there was an unused key!
if len(unuusedContext) != 0 {
keys := maps.Keys(unuusedContext)
// Check if there were missing template keys.
// If so, we sort them and return an appropriate error.
if len(missingKeys) != 0 {
keys := maps.Keys(unusedKeys)
slices.Sort(keys)
return errors.Errorf("additional keys %s in context", strings.Join(keys, ","))
return MissingTemplateKeyError{
Keys: keys,
}
}
// Check if there were unused template keys.
// If so, we sort them and return an appropriate error.
if len(unusedKeys) != 0 {
keys := maps.Keys(unusedKeys)
slices.Sort(keys)
return UnusuedTemplateKeyError{
Keys: keys,
}
}
return nil

65
internal/unpack/unpack.go Normal file
View file

@ -0,0 +1,65 @@
// Package unpack unpacks files and templates to a target directory
package unpack
import (
"bytes"
"errors"
"io/fs"
)
var errExpectedFileButGotDirectory = errors.New("expected a file, but got a directory")
var errExpectedDirectoryButGotFile = errors.New("expected a directory, but got a file")
// InstallResource installs the resource at src within fsys to dst.
//
// OnInstallFile is called for each source and destination file.
// OnInstallFile may be nil.
//
// See [InstallDir] or [InstallFile].
func InstallResource(dst string, src string, fsys fs.FS, onInstallFile func(dst, src string)) error {
// open the srcFile
srcFile, err := fsys.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
// stat it!
srcInfo, err := srcFile.Stat()
if err != nil {
return err
}
// call the hook (if any)
if onInstallFile != nil {
onInstallFile(dst, src)
}
// this is a directory, so the cast is safe!
if srcInfo.IsDir() {
return installDir(dst, srcInfo, srcFile.(fs.ReadDirFile), src, fsys, onInstallFile)
}
// this is a regular file!
return installFile(dst, srcInfo, srcFile)
}
// UnpackTemplate unpacks the given file template and template.
// See [WriteTemplate] for possible errors.
func UnpackTemplate(context map[string]string, src fs.File) ([]byte, fs.FileMode, error) {
// stat the source file to install
srcStat, srcErr := src.Stat()
if srcErr != nil {
return nil, 0, srcErr
}
// should not be a directory
if srcStat.IsDir() {
return nil, 0, errExpectedFileButGotDirectory
}
// read all the bytes into a buffer
var buffer bytes.Buffer
err := WriteTemplate(&buffer, context, src)
return buffer.Bytes(), srcStat.Mode(), err
}