From b5bf4ef31a8fb12d8c02c22f6ea8ab69a2693765 Mon Sep 17 00:00:00 2001 From: Johnny Date: Thu, 19 Aug 2021 10:28:13 +0100 Subject: [PATCH] interp: allow for reading source files from diverse filesystems Make use of fs.FS (new to go 1.16) to allow for reading source files from diverse filesystems (local, embed, custom). * `Options` has a new field `SourcecodeFilesystem fs.FS` so users can supply their own read-only filesystem containing source code. * Defaults to the local filesystems (via `RealFS` - a thin `os.Open` wrapper complying with `fs.FS`) so regular users should see no change in behaviour. * When no filesystem is available (e.g. WASM, or if you want to embed files to retain single binary distribution) an alternative filesystem is preferable to using `Eval(string)` as that requires the stringy code to be a single file monolith instead of multiple files. By using an `fs.FS` we can use `EvalPath()` and gain the ability to handle multiple files and packages. * You can make use of embed filesystems (https://pkg.go.dev/embed) and custom filesystems obeying the `fs.FS` interface (I use one for http served zip files when targeting wasm as there is no local filesystem on wasm). Tests can make use of `fstest.Map`. * NOTE: This does NOT affect what the running yaegi code considers its local filesystem, this is only for the interpreter finding the source code. See `example/fs/fs_test.go` for an example. Fixes #1200. --- example/fs/fs_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- interp/interp.go | 23 +++++++++++---- interp/realfs.go | 21 ++++++++++++++ interp/src.go | 16 +++++------ interp/src_test.go | 8 ++++-- 6 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 example/fs/fs_test.go create mode 100644 interp/realfs.go diff --git a/example/fs/fs_test.go b/example/fs/fs_test.go new file mode 100644 index 00000000..040d05e1 --- /dev/null +++ b/example/fs/fs_test.go @@ -0,0 +1,67 @@ +package fs1 + +import ( + "testing" + + // only available from 1.16. + "testing/fstest" + + "github.com/traefik/yaegi/interp" + "github.com/traefik/yaegi/stdlib" +) + +var testFilesystem = fstest.MapFS{ + "main.go": &fstest.MapFile{ + Data: []byte(`package main + +import ( + "foo/bar" + "./localfoo" +) + +func main() { + bar.PrintSomething() + localfoo.PrintSomethingElse() +} +`), + }, + "_pkg/src/foo/bar/bar.go": &fstest.MapFile{ + Data: []byte(`package bar + +import ( + "fmt" +) + +func PrintSomething() { + fmt.Println("I am a virtual filesystem printing something from _pkg/src/foo/bar/bar.go!") +} +`), + }, + "localfoo/foo.go": &fstest.MapFile{ + Data: []byte(`package localfoo + +import ( + "fmt" +) + +func PrintSomethingElse() { + fmt.Println("I am virtual filesystem printing else from localfoo/foo.go!") +} +`), + }, +} + +func TestFilesystemMapFS(t *testing.T) { + i := interp.New(interp.Options{ + GoPath: "./_pkg", + SourcecodeFilesystem: testFilesystem, + }) + if err := i.Use(stdlib.Symbols); err != nil { + t.Fatal(err) + } + + _, err := i.EvalPath(`main.go`) + if err != nil { + t.Fatal(err) + } +} diff --git a/go.mod b/go.mod index cca7660d..ba0618d5 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/traefik/yaegi -go 1.12 +go 1.16 diff --git a/interp/interp.go b/interp/interp.go index d97b4a7f..8109fba0 100644 --- a/interp/interp.go +++ b/interp/interp.go @@ -10,7 +10,7 @@ import ( "go/scanner" "go/token" "io" - "io/ioutil" + "io/fs" "log" "os" "os/signal" @@ -136,6 +136,7 @@ type opt struct { stdin io.Reader // standard input stdout io.Writer // standard output stderr io.Writer // standard error + filesystem fs.FS } // Interpreter contains global resources and state. @@ -253,12 +254,18 @@ type Options struct { // They default to os.Stdin, os.Stdout and os.Stderr respectively. Stdin io.Reader Stdout, Stderr io.Writer + + // SourcecodeFilesystem is where the _sourcecode_ is loaded from and does + // NOT affect the filesystem of scripts when they run. + // It can be any fs.FS compliant filesystem (e.g. embed.FS, or fstest.MapFS for testing) + // See example/fs/fs_test.go for an example. + SourcecodeFilesystem fs.FS } // New returns a new interpreter. func New(options Options) *Interpreter { i := Interpreter{ - opt: opt{context: build.Default}, + opt: opt{context: build.Default, filesystem: &RealFS{}}, frame: newFrame(nil, 0, 0), fset: token.NewFileSet(), universe: initUniverse(), @@ -282,6 +289,10 @@ func New(options Options) *Interpreter { i.opt.stderr = os.Stderr } + if options.SourcecodeFilesystem != nil { + i.opt.filesystem = options.SourcecodeFilesystem + } + i.opt.context.GOPATH = options.GoPath if len(options.BuildTags) > 0 { i.opt.context.BuildTags = options.BuildTags @@ -407,12 +418,12 @@ func (interp *Interpreter) Eval(src string) (res reflect.Value, err error) { // by the interpreter, and a non nil error in case of failure. // The main function of the main package is executed if present. func (interp *Interpreter) EvalPath(path string) (res reflect.Value, err error) { - if !isFile(path) { + if !isFile(interp.opt.filesystem, path) { _, err := interp.importSrc(mainID, path, NoTest) return res, err } - b, err := ioutil.ReadFile(path) + b, err := fs.ReadFile(interp.filesystem, path) if err != nil { return res, err } @@ -484,8 +495,8 @@ func (interp *Interpreter) Symbols(importPath string) Exports { return m } -func isFile(path string) bool { - fi, err := os.Stat(path) +func isFile(filesystem fs.FS, path string) bool { + fi, err := fs.Stat(filesystem, path) return err == nil && fi.Mode().IsRegular() } diff --git a/interp/realfs.go b/interp/realfs.go new file mode 100644 index 00000000..325ae413 --- /dev/null +++ b/interp/realfs.go @@ -0,0 +1,21 @@ +package interp + +import ( + "io/fs" + "os" +) + +// RealFS complies with the fs.FS interface (go 1.16 onwards) +// We use this rather than os.DirFS as DirFS has no concept of +// what the current working directory is, whereas this simple +// passthru to os.Open knows about working dir automagically. +type RealFS struct{} + +// Open complies with the fs.FS interface. +func (dir RealFS) Open(name string) (fs.File, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + return f, nil +} diff --git a/interp/src.go b/interp/src.go index fdc1aed9..2d2c00a5 100644 --- a/interp/src.go +++ b/interp/src.go @@ -2,7 +2,7 @@ package interp import ( "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" @@ -48,7 +48,7 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s } interp.rdir[importPath] = true - files, err := ioutil.ReadDir(dir) + files, err := fs.ReadDir(interp.opt.filesystem, dir) if err != nil { return "", err } @@ -69,7 +69,7 @@ func (interp *Interpreter) importSrc(rPath, importPath string, skipTest bool) (s name = filepath.Join(dir, name) var buf []byte - if buf, err = ioutil.ReadFile(name); err != nil { + if buf, err = fs.ReadFile(interp.opt.filesystem, name); err != nil { return "", err } @@ -185,13 +185,13 @@ func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (strin rPath := filepath.Join(root, "vendor") dir := filepath.Join(goPath, "src", rPath, importPath) - if _, err := os.Stat(dir); err == nil { + if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil { return dir, rPath, nil // found! } dir = filepath.Join(goPath, "src", effectivePkg(root, importPath)) - if _, err := os.Stat(dir); err == nil { + if _, err := fs.Stat(interp.opt.filesystem, dir); err == nil { return dir, root, nil // found! } @@ -203,7 +203,7 @@ func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (strin } rootPath := filepath.Join(goPath, "src", root) - prevRoot, err := previousRoot(rootPath, root) + prevRoot, err := previousRoot(interp.opt.filesystem, rootPath, root) if err != nil { return "", "", err } @@ -214,7 +214,7 @@ func (interp *Interpreter) pkgDir(goPath string, root, importPath string) (strin const vendor = "vendor" // Find the previous source root (vendor > vendor > ... > GOPATH). -func previousRoot(rootPath, root string) (string, error) { +func previousRoot(filesystem fs.FS, rootPath, root string) (string, error) { rootPath = filepath.Clean(rootPath) parent, final := filepath.Split(rootPath) parent = filepath.Clean(parent) @@ -227,7 +227,7 @@ func previousRoot(rootPath, root string) (string, error) { // look for the closest vendor in one of our direct ancestors, as it takes priority. var vendored string for { - fi, err := os.Lstat(filepath.Join(parent, vendor)) + fi, err := fs.Stat(filesystem, filepath.Join(parent, vendor)) if err == nil && fi.IsDir() { vendored = strings.TrimPrefix(strings.TrimPrefix(parent, prefix), string(filepath.Separator)) break diff --git a/interp/src_test.go b/interp/src_test.go index 3a9d52db..f9490cc8 100644 --- a/interp/src_test.go +++ b/interp/src_test.go @@ -161,7 +161,11 @@ func Test_pkgDir(t *testing.T) { }, } - interp := &Interpreter{} + interp := &Interpreter{ + opt: opt{ + filesystem: &RealFS{}, + }, + } for _, test := range testCases { test := test @@ -247,7 +251,7 @@ func Test_previousRoot(t *testing.T) { } else { rootPath = vendor } - p, err := previousRoot(rootPath, test.root) + p, err := previousRoot(&RealFS{}, rootPath, test.root) if err != nil { t.Error(err) }