diff --git a/compiler.go b/compiler.go index f50908d7..e4e04303 100644 --- a/compiler.go +++ b/compiler.go @@ -1,9 +1,11 @@ package tengo import ( + "errors" "fmt" "io" "io/ioutil" + "os" "path/filepath" "reflect" "strings" @@ -45,6 +47,7 @@ type Compiler struct { parent *Compiler modulePath string importDir string + importFileExt []string constants []Object symbolTable *SymbolTable scopes []compilationScope @@ -96,6 +99,7 @@ func NewCompiler( trace: trace, modules: modules, compiledModules: make(map[string]*CompiledFunction), + importFileExt: []string{SourceFileExtDefault}, } } @@ -538,12 +542,8 @@ func (c *Compiler) Compile(node parser.Node) error { } } else if c.allowFileImport { moduleName := node.ModuleName - if !strings.HasSuffix(moduleName, ".tengo") { - moduleName += ".tengo" - } - modulePath, err := filepath.Abs( - filepath.Join(c.importDir, moduleName)) + modulePath, err := c.getPathModule(moduleName) if err != nil { return c.errorf(node, "module file path error: %s", err.Error()) @@ -640,6 +640,39 @@ func (c *Compiler) SetImportDir(dir string) { c.importDir = dir } +// SetImportFileExt sets the extension name of the source file for loading +// local module files. +// +// Use this method if you want other source file extension than ".tengo". +// +// // this will search for *.tengo, *.foo, *.bar +// err := c.SetImportFileExt(".tengo", ".foo", ".bar") +// +// This function requires at least one argument, since it will replace the +// current list of extension name. +func (c *Compiler) SetImportFileExt(exts ...string) error { + if len(exts) == 0 { + return fmt.Errorf("missing arg: at least one argument is required") + } + + for _, ext := range exts { + if ext != filepath.Ext(ext) || ext == "" { + return fmt.Errorf("invalid file extension: %s", ext) + } + } + + c.importFileExt = exts // Replace the hole current extension list + + return nil +} + +// GetImportFileExt returns the current list of extension name. +// Thease are the complementary suffix of the source file to search and load +// local module files. +func (c *Compiler) GetImportFileExt() []string { + return c.importFileExt +} + func (c *Compiler) compileAssign( node parser.Node, lhs, rhs []parser.Expr, @@ -1098,6 +1131,7 @@ func (c *Compiler) fork( child.parent = c // parent to set to current compiler child.allowFileImport = c.allowFileImport child.importDir = c.importDir + child.importFileExt = c.importFileExt if isFile && c.importDir != "" { child.importDir = filepath.Dir(modulePath) } @@ -1287,6 +1321,28 @@ func (c *Compiler) printTrace(a ...interface{}) { _, _ = fmt.Fprintln(c.trace, a...) } +func (c *Compiler) getPathModule(moduleName string) (pathFile string, err error) { + for _, ext := range c.importFileExt { + nameFile := moduleName + + if !strings.HasSuffix(nameFile, ext) { + nameFile += ext + } + + pathFile, err = filepath.Abs(filepath.Join(c.importDir, nameFile)) + if err != nil { + continue + } + + // Check if file exists + if _, err := os.Stat(pathFile); !errors.Is(err, os.ErrNotExist) { + return pathFile, nil + } + } + + return "", fmt.Errorf("module '%s' not found at: %s", moduleName, pathFile) +} + func resolveAssignLHS( expr parser.Expr, ) (name string, selectors []parser.Expr) { diff --git a/compiler_test.go b/compiler_test.go index 0fe93121..21d1f02d 100644 --- a/compiler_test.go +++ b/compiler_test.go @@ -2,12 +2,15 @@ package tengo_test import ( "fmt" + "io/ioutil" + "path/filepath" "strings" "testing" "github.com/d5/tengo/v2" "github.com/d5/tengo/v2/parser" "github.com/d5/tengo/v2/require" + "github.com/d5/tengo/v2/stdlib" ) func TestCompiler_Compile(t *testing.T) { @@ -1010,7 +1013,7 @@ r["x"] = { expectCompileError(t, ` (func() { fn := fn() -})() +})() `, "unresolved reference 'fn") } @@ -1222,6 +1225,91 @@ func() { tengo.MakeInstruction(parser.OpReturn, 0))))) } +func TestCompiler_custom_extension(t *testing.T) { + pathFileSource := "./testdata/issue286/test.mshk" + + modules := stdlib.GetModuleMap(stdlib.AllModuleNames()...) + + src, err := ioutil.ReadFile(pathFileSource) + require.NoError(t, err) + + // Escape shegang + if len(src) > 1 && string(src[:2]) == "#!" { + copy(src, "//") + } + + fileSet := parser.NewFileSet() + srcFile := fileSet.AddFile(filepath.Base(pathFileSource), -1, len(src)) + + p := parser.NewParser(srcFile, src, nil) + file, err := p.ParseFile() + require.NoError(t, err) + + c := tengo.NewCompiler(srcFile, nil, nil, modules, nil) + c.EnableFileImport(true) + c.SetImportDir(filepath.Dir(pathFileSource)) + + // Search for "*.tengo" and ".mshk"(custom extension) + c.SetImportFileExt(".tengo", ".mshk") + + err = c.Compile(file) + require.NoError(t, err) +} + +func TestCompilerNewCompiler_default_file_extension(t *testing.T) { + modules := stdlib.GetModuleMap(stdlib.AllModuleNames()...) + input := "{}" + fileSet := parser.NewFileSet() + file := fileSet.AddFile("test", -1, len(input)) + + c := tengo.NewCompiler(file, nil, nil, modules, nil) + c.EnableFileImport(true) + + require.Equal(t, []string{".tengo"}, c.GetImportFileExt(), + "newly created compiler object must contain the default extension") +} + +func TestCompilerSetImportExt_extension_name_validation(t *testing.T) { + c := new(tengo.Compiler) // Instantiate a new compiler object with no initialization + + // Test of empty arg + err := c.SetImportFileExt() + + require.Error(t, err, "empty arg should return an error") + + // Test of various arg types + for _, test := range []struct { + extensions []string + expect []string + requireErr bool + msgFail string + }{ + {[]string{".tengo"}, []string{".tengo"}, false, + "well-formed extension should not return an error"}, + {[]string{""}, []string{".tengo"}, true, + "empty extension name should return an error"}, + {[]string{"foo"}, []string{".tengo"}, true, + "name without dot prefix should return an error"}, + {[]string{"foo.bar"}, []string{".tengo"}, true, + "malformed extension should return an error"}, + {[]string{"foo."}, []string{".tengo"}, true, + "malformed extension should return an error"}, + {[]string{".mshk"}, []string{".mshk"}, false, + "name with dot prefix should be added"}, + {[]string{".foo", ".bar"}, []string{".foo", ".bar"}, false, + "it should replace instead of appending"}, + } { + err := c.SetImportFileExt(test.extensions...) + if test.requireErr { + require.Error(t, err, test.msgFail) + } + + expect := test.expect + actual := c.GetImportFileExt() + require.Equal(t, expect, actual, test.msgFail) + } +} + func concatInsts(instructions ...[]byte) []byte { var concat []byte for _, i := range instructions { diff --git a/docs/tutorial.md b/docs/tutorial.md index 9f98358b..92c7a3ce 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -37,7 +37,7 @@ Here's a list of all available value types in Tengo. | map | value map with string keys _(mutable)_ | `map[string]interface{}` | | immutable map | [immutable](#immutable-values) map | - | | undefined | [undefined](#undefined-values) value | - | -| function | [function](#function-values) value | - | +| function | [function](#function-values) value | - | | _user-defined_ | value of [user-defined types](https://github.com/d5/tengo/blob/master/docs/objects.md) | - | ### Error Values @@ -45,14 +45,14 @@ Here's a list of all available value types in Tengo. In Tengo, an error can be represented using "error" typed values. An error value is created using `error` expression, and, it must have an underlying value. The underlying value of an error value can be access using `.value` -selector. +selector. ```golang err1 := error("oops") // error with string value err2 := error(1+2+3) // error with int value if is_error(err1) { // 'is_error' builtin function err_val := err1.value // get underlying value -} +} ``` ### Immutable Values @@ -101,12 +101,12 @@ a.c[1] = 5 // illegal ### Undefined Values In Tengo, an "undefined" value can be used to represent an unexpected or -non-existing value: +non-existing value: - A function that does not return a value explicitly considered to return `undefined` value. - Indexer or selector on composite value types may return `undefined` if the -key or index does not exist. +key or index does not exist. - Type conversion builtin functions without a default value will return `undefined` if conversion fails. @@ -142,8 +142,8 @@ m["b"] // == false m.c // == "foo" m.x // == undefined -{a: [1,2,3], b: {c: "foo", d: "bar"}} // ok: map with an array element and a map element -``` +{a: [1,2,3], b: {c: "foo", d: "bar"}} // ok: map with an array element and a map element +``` ### Function Values @@ -233,7 +233,7 @@ a := "foo" // define 'a' in global scope func() { // function scope A b := 52 // define 'b' in function scope A - + func() { // function scope B c := 19.84 // define 'c' in function scope B @@ -243,12 +243,12 @@ func() { // function scope A b := true // ok: define new 'b' in function scope B // (shadowing 'b' from function scope A) } - + a = "bar" // ok: assigne new value to 'a' from global scope b = 10 // ok: assigne new value to 'b' a := -100 // ok: define new 'a' in function scope A // (shadowing 'a' from global scope) - + c = -9.1 // illegal: 'c' is not defined b := [1, 2] // illegal: 'b' is already defined in the same scope } @@ -470,7 +470,7 @@ for { "For-In" statement is new in Tengo. It's similar to Go's `for range` statement. "For-In" statement can iterate any iterable value types (array, map, bytes, -string, undefined). +string, undefined). ```golang for v in [1, 2, 3] { // array: element @@ -478,7 +478,7 @@ for v in [1, 2, 3] { // array: element } for i, v in [1, 2, 3] { // array: index and element // 'i' is index - // 'v' is value + // 'v' is value } for k, v in {k1: 1, k2: 2} { // map: key and value // 'k' is key @@ -508,6 +508,16 @@ export func(x) { } ``` +By default, `import` solves the missing extension name of a module file as +"`.tengo`"[^note]. +Thus, `sum := import("./sum")` is equivalent to `sum := import("./sum.tengo")`. + +[^note]: + If using Tengo as a library in Go, the file extension name "`.tengo`" can + be customized. In that case, use the `SetImportFileExt` function of the + `Compiler` type. + See the [Go reference](https://pkg.go.dev/github.com/d5/tengo/v2) for details. + In Tengo, modules are very similar to functions. - `import` expression loads the module code and execute it like a function. @@ -517,9 +527,9 @@ In Tengo, modules are very similar to functions. return a value to the importing code. - `export`-ed values are always immutable. - If the module does not have any `export` statement, `import` expression - simply returns `undefined`. _(Just like the function that has no `return`.)_ + simply returns `undefined`. _(Just like the function that has no `return`.)_ - Note that `export` statement is completely ignored and not evaluated if - the code is executed as a main module. + the code is executed as a main module. Also, you can use `import` expression to load the [Standard Library](https://github.com/d5/tengo/blob/master/docs/stdlib.md) as diff --git a/tengo.go b/tengo.go index 098a1970..490e9aed 100644 --- a/tengo.go +++ b/tengo.go @@ -26,6 +26,9 @@ const ( // MaxFrames is the maximum number of function frames for a VM. MaxFrames = 1024 + + // SourceFileExtDefault is the default extension for source files. + SourceFileExtDefault = ".tengo" ) // CallableFunc is a function signature for the callable functions. diff --git a/testdata/issue286/dos/cinco/cinco.mshk b/testdata/issue286/dos/cinco/cinco.mshk new file mode 100644 index 00000000..7232e827 --- /dev/null +++ b/testdata/issue286/dos/cinco/cinco.mshk @@ -0,0 +1,8 @@ +export { + fn: func(...args) { + text := import("text") + args = append(args, "cinco") + + return text.join(args, " ") + } +} \ No newline at end of file diff --git a/testdata/issue286/dos/dos.mshk b/testdata/issue286/dos/dos.mshk new file mode 100644 index 00000000..62efdefa --- /dev/null +++ b/testdata/issue286/dos/dos.mshk @@ -0,0 +1,7 @@ +export { + fn: func(a, b) { + tres := import("../tres") + + return tres.fn(a, b, "dos") + } +} \ No newline at end of file diff --git a/testdata/issue286/dos/quatro/quatro.mshk b/testdata/issue286/dos/quatro/quatro.mshk new file mode 100644 index 00000000..90cf5ebd --- /dev/null +++ b/testdata/issue286/dos/quatro/quatro.mshk @@ -0,0 +1,7 @@ +export { + fn: func(a, b, c, d) { + cinco := import("../cinco/cinco") + + return cinco.fn(a, b, c, d, "quatro") + } +} \ No newline at end of file diff --git a/testdata/issue286/test.mshk b/testdata/issue286/test.mshk new file mode 100644 index 00000000..4a98a2b2 --- /dev/null +++ b/testdata/issue286/test.mshk @@ -0,0 +1,23 @@ +#!/usr/bin/env tengo +// This is a test of custom extension for issue #286 and PR #350. +// Which allows the tengo library to use custom extension names for the +// source files. +// +// This test should pass if the interpreter's tengo.Compiler.SetImportExt() +// was set as `c.SetImportExt(".tengo", ".mshk")`. + +os := import("os") +uno := import("uno") // it will search uno.tengo and uno.mshk +fmt := import("fmt") +text := import("text") + +expected := ["test", "uno", "dos", "tres", "quatro", "cinco"] +expected = text.join(expected, " ") +if v := uno.fn("test"); v != expected { + fmt.printf("relative import test error:\n\texpected: %v\n\tgot : %v\n", + expected, v) + os.exit(1) +} + +args := text.join(os.args(), " ") +fmt.println("ok\t", args) diff --git a/testdata/issue286/tres.tengo b/testdata/issue286/tres.tengo new file mode 100644 index 00000000..d13dddd0 --- /dev/null +++ b/testdata/issue286/tres.tengo @@ -0,0 +1,6 @@ +export { + fn: func(a, b, c) { + quatro := import("./dos/quatro/quatro.mshk") + return quatro.fn(a, b, c, "tres") + } +} \ No newline at end of file diff --git a/testdata/issue286/uno.mshk b/testdata/issue286/uno.mshk new file mode 100644 index 00000000..94789760 --- /dev/null +++ b/testdata/issue286/uno.mshk @@ -0,0 +1,6 @@ +export { + fn: func(a) { + dos := import("dos/dos") + return dos.fn(a, "uno") + } +} \ No newline at end of file