Merge pull request #74 from d5/iarrlen

Some bug fixes

assert/assert.go

@@ -121,8 +121,8 @@ func Equal(t *testing.T, expected, actual interface{}, msg ...interface{}) bool
 		if expected != actual.(rune) {
 			return failExpectedActual(t, expected, actual, msg...)
 		}
-	case compiler.Symbol:
-		if !equalSymbol(expected, actual.(compiler.Symbol)) {
+	case *compiler.Symbol:
+		if !equalSymbol(expected, actual.(*compiler.Symbol)) {
 			return failExpectedActual(t, expected, actual, msg...)
 		}
 	case source.Pos:
@@ -238,7 +238,7 @@ func equalIntSlice(a, b []int) bool {
 	return true
 }
 
-func equalSymbol(a, b compiler.Symbol) bool {
+func equalSymbol(a, b *compiler.Symbol) bool {
 	return a.Name == b.Name &&
 		a.Index == b.Index &&
 		a.Scope == b.Scope

compiler/ast/export_stmt.go

@@ -0,0 +1,27 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+)
+
+// ExportStmt represents an export statement.
+type ExportStmt struct {
+	ExportPos source.Pos
+	Result    Expr
+}
+
+func (s *ExportStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *ExportStmt) Pos() source.Pos {
+	return s.ExportPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *ExportStmt) End() source.Pos {
+	return s.Result.End()
+}
+
+func (s *ExportStmt) String() string {
+	return "export " + s.Result.String()
+}

compiler/compilation_scope.go

@@ -5,4 +5,5 @@ package compiler
 type CompilationScope struct {
 	instructions     []byte
 	lastInstructions [2]EmittedInstruction
+	symbolInit       map[string]bool
 }

compiler/compiler.go

@@ -21,7 +21,7 @@ type Compiler struct {
 	scopeIndex      int
 	moduleLoader    ModuleLoader
 	stdModules      map[string]*objects.ImmutableMap
-	compiledModules map[string]*objects.CompiledModule
+	compiledModules map[string]*objects.CompiledFunction
 	loops           []*Loop
 	loopIndex       int
 	trace           io.Writer
@@ -60,7 +60,7 @@ func NewCompiler(symbolTable *SymbolTable, stdModules map[string]*objects.Immuta
 		loopIndex:       -1,
 		trace:           trace,
 		stdModules:      stdModules,
-		compiledModules: make(map[string]*objects.CompiledModule),
+		compiledModules: make(map[string]*objects.CompiledFunction),
 	}
 }
 
@@ -383,7 +383,10 @@ func (c *Compiler) Compile(node ast.Node) error {
 		c.enterScope()
 
 		for _, p := range node.Type.Params.List {
-			c.symbolTable.Define(p.Name)
+			s := c.symbolTable.Define(p.Name)
+
+			// function arguments is not assigned directly.
+			s.LocalAssigned = true
 		}
 
 		if err := c.Compile(node.Body); err != nil {
@@ -402,6 +405,50 @@ func (c *Compiler) Compile(node ast.Node) error {
 		for _, s := range freeSymbols {
 			switch s.Scope {
 			case ScopeLocal:
+				if !s.LocalAssigned {
+					// Here, the closure is capturing a local variable that's not yet assigned its value.
+					// One example is a local recursive function:
+					//
+					//   func() {
+					//     foo := func(x) {
+					//       // ..
+					//       return foo(x-1)
+					//     }
+					//   }
+					//
+					// which translate into
+					//
+					//   0000 GETL    0
+					//   0002 CLOSURE ?     1
+					//   0006 DEFL    0
+					//
+					// . So the local variable (0) is being captured before it's assigned the value.
+					//
+					// Solution is to transform the code into something like this:
+					//
+					//   func() {
+					//     foo := undefined
+					//     foo = func(x) {
+					//       // ..
+					//       return foo(x-1)
+					//     }
+					//   }
+					//
+					// that is equivalent to
+					//
+					//   0000 NULL
+					//   0001 DEFL    0
+					//   0003 GETL    0
+					//   0005 CLOSURE ?     1
+					//   0009 SETL    0
+					//
+
+					c.emit(OpNull)
+					c.emit(OpDefineLocal, s.Index)
+
+					s.LocalAssigned = true
+				}
+
 				c.emit(OpGetLocal, s.Index)
 			case ScopeFree:
 				c.emit(OpGetFree, s.Index)
@@ -461,9 +508,28 @@ func (c *Compiler) Compile(node ast.Node) error {
 				return err
 			}
 
-			c.emit(OpModule, c.addConstant(userMod))
+			c.emit(OpConstant, c.addConstant(userMod))
+			c.emit(OpCall, 0)
+		}
+
+	case *ast.ExportStmt:
+		// export statement must be in top-level scope
+		if c.scopeIndex != 0 {
+			return fmt.Errorf("cannot use 'export' inside function")
+		}
+
+		// export statement is simply ignore when compiling non-module code
+		if c.parent == nil {
+			break
 		}
 
+		if err := c.Compile(node.Result); err != nil {
+			return err
+		}
+
+		c.emit(OpImmutable)
+		c.emit(OpReturnValue)
+
 	case *ast.ErrorExpr:
 		if err := c.Compile(node.Expr); err != nil {
 			return err

compiler/compiler_assign.go

@@ -105,12 +105,15 @@ func (c *Compiler) compileAssign(lhs, rhs []ast.Expr, op token.Token) error {
 		if numSel > 0 {
 			c.emit(OpSetSelLocal, symbol.Index, numSel)
 		} else {
-			if op == token.Define {
+			if op == token.Define && !symbol.LocalAssigned {
 				c.emit(OpDefineLocal, symbol.Index)
 			} else {
 				c.emit(OpSetLocal, symbol.Index)
 			}
 		}
+
+		// mark the symbol as local-assigned
+		symbol.LocalAssigned = true
 	case ScopeFree:
 		if numSel > 0 {
 			c.emit(OpSetSelFree, symbol.Index, numSel)

compiler/compiler_module.go

@@ -14,7 +14,7 @@ var (
 	fileSet = source.NewFileSet()
 )
 
-func (c *Compiler) compileModule(moduleName string) (*objects.CompiledModule, error) {
+func (c *Compiler) compileModule(moduleName string) (*objects.CompiledFunction, error) {
 	compiledModule, exists := c.loadCompiledModule(moduleName)
 	if exists {
 		return compiledModule, nil
@@ -69,35 +69,44 @@ func (c *Compiler) checkCyclicImports(moduleName string) error {
 	return nil
 }
 
-func (c *Compiler) doCompileModule(moduleName string, src []byte) (*objects.CompiledModule, error) {
+func (c *Compiler) doCompileModule(moduleName string, src []byte) (*objects.CompiledFunction, error) {
 	p := parser.NewParser(fileSet.AddFile(moduleName, -1, len(src)), src, nil)
 	file, err := p.ParseFile()
 	if err != nil {
 		return nil, err
 	}
 
 	symbolTable := NewSymbolTable()
-	globals := make(map[string]int)
 
+	// inherit builtin functions
+	for idx, fn := range objects.Builtins {
+		s, _, ok := c.symbolTable.Resolve(fn.Name)
+		if ok && s.Scope == ScopeBuiltin {
+			symbolTable.DefineBuiltin(idx, fn.Name)
+		}
+	}
+
+	// no global scope for the module
+	symbolTable = symbolTable.Fork(false)
+
+	// compile module
 	moduleCompiler := c.fork(moduleName, symbolTable)
 	if err := moduleCompiler.Compile(file); err != nil {
 		return nil, err
 	}
 
-	for _, name := range symbolTable.Names() {
-		symbol, _, _ := symbolTable.Resolve(name)
-		if symbol.Scope == ScopeGlobal {
-			globals[name] = symbol.Index
-		}
+	// add OpReturn (== export undefined) if export is missing
+	if !moduleCompiler.lastInstructionIs(OpReturnValue) {
+		moduleCompiler.emit(OpReturn)
 	}
 
-	return &objects.CompiledModule{
+	return &objects.CompiledFunction{
 		Instructions: moduleCompiler.Bytecode().Instructions,
-		Globals:      globals,
+		NumLocals:    symbolTable.MaxSymbols(),
 	}, nil
 }
 
-func (c *Compiler) loadCompiledModule(moduleName string) (mod *objects.CompiledModule, ok bool) {
+func (c *Compiler) loadCompiledModule(moduleName string) (mod *objects.CompiledFunction, ok bool) {
 	if c.parent != nil {
 		return c.parent.loadCompiledModule(moduleName)
 	}
@@ -107,7 +116,7 @@ func (c *Compiler) loadCompiledModule(moduleName string) (mod *objects.CompiledM
 	return
 }
 
-func (c *Compiler) storeCompiledModule(moduleName string, module *objects.CompiledModule) {
+func (c *Compiler) storeCompiledModule(moduleName string, module *objects.CompiledFunction) {
 	if c.parent != nil {
 		c.parent.storeCompiledModule(moduleName, module)
 	}

compiler/compiler_scopes.go

@@ -7,6 +7,7 @@ func (c *Compiler) currentInstructions() []byte {
 func (c *Compiler) enterScope() {
 	scope := CompilationScope{
 		instructions: make([]byte, 0),
+		symbolInit:   make(map[string]bool),
 	}
 
 	c.scopes = append(c.scopes, scope)

compiler/opcodes.go

@@ -41,6 +41,7 @@ const (
 	OpCall                           // Call function
 	OpReturn                         // Return
 	OpReturnValue                    // Return value
+	OpExport                         // Export
 	OpGetGlobal                      // Get global variable
 	OpSetGlobal                      // Set global variable
 	OpSetSelGlobal                   // Set global variable using selectors
@@ -57,7 +58,6 @@ const (
 	OpIteratorNext                   // Iterator next
 	OpIteratorKey                    // Iterator key
 	OpIteratorValue                  // Iterator value
-	OpModule                         // Module
 )
 
 // OpcodeNames is opcode names.
@@ -101,6 +101,7 @@ var OpcodeNames = [...]string{
 	OpCall:             "CALL",
 	OpReturn:           "RET",
 	OpReturnValue:      "RETVAL",
+	OpExport:           "EXPORT",
 	OpGetLocal:         "GETL",
 	OpSetLocal:         "SETL",
 	OpDefineLocal:      "DEFL",
@@ -114,7 +115,6 @@ var OpcodeNames = [...]string{
 	OpIteratorNext:     "ITNXT",
 	OpIteratorKey:      "ITKEY",
 	OpIteratorValue:    "ITVAL",
-	OpModule:           "MODULE",
 }
 
 // OpcodeOperands is the number of operands.
@@ -158,6 +158,7 @@ var OpcodeOperands = [...][]int{
 	OpCall:             {1},
 	OpReturn:           {},
 	OpReturnValue:      {},
+	OpExport:           {},
 	OpGetLocal:         {1},
 	OpSetLocal:         {1},
 	OpDefineLocal:      {1},
@@ -171,7 +172,6 @@ var OpcodeOperands = [...][]int{
 	OpIteratorNext:     {},
 	OpIteratorKey:      {},
 	OpIteratorValue:    {},
-	OpModule:           {2},
 }
 
 // ReadOperands reads operands from the bytecode.

compiler/parser/parser.go

@@ -628,6 +628,8 @@ func (p *Parser) parseStmt() (stmt ast.Stmt) {
 		return s
 	case token.Return:
 		return p.parseReturnStmt()
+	case token.Export:
+		return p.parseExportStmt()
 	case token.If:
 		return p.parseIfStmt()
 	case token.For:
@@ -874,6 +876,23 @@ func (p *Parser) parseReturnStmt() ast.Stmt {
 	}
 }
 
+func (p *Parser) parseExportStmt() ast.Stmt {
+	if p.trace {
+		defer un(trace(p, "ExportStmt"))
+	}
+
+	pos := p.pos
+	p.expect(token.Export)
+
+	x := p.parseExpr()
+	p.expectSemi()
+
+	return &ast.ExportStmt{
+		ExportPos: pos,
+		Result:    x,
+	}
+}
+
 func (p *Parser) parseSimpleStmt(forIn bool) ast.Stmt {
 	if p.trace {
 		defer un(trace(p, "SimpleStmt"))

compiler/parser/parser_function_test.go

@@ -8,23 +8,6 @@ import (
 )
 
 func TestFunction(t *testing.T) {
-	// TODO: function declaration currently not parsed.
-	// All functions are parsed as function literal instead.
-	// In Go, function declaration is parsed only at the top level.
-	//expect(t, "func a(b, c, d) {}", func(p pfn) []ast.Stmt {
-	//	return stmts(
-	//		declStmt(
-	//			funcDecl(
-	//				ident("a", p(1, 6)),
-	//				funcType(
-	//					identList(p(1, 7), p(1, 15),
-	//						ident("b", p(1, 8)),
-	//						ident("c", p(1, 11)),
-	//						ident("d", p(1, 14))),
-	//					p(1, 12)),
-	//				blockStmt(p(1, 17), p(1, 18)))))
-	//})
-
 	expect(t, "a = func(b, c, d) { return d }", func(p pfn) []ast.Stmt {
 		return stmts(
 			assignStmt(

compiler/parser/sync.go

@@ -8,4 +8,5 @@ var stmtStart = map[token.Token]bool{
 	token.For:      true,
 	token.If:       true,
 	token.Return:   true,
+	token.Export:   true,
 }

compiler/scanner/scanner.go

@@ -77,7 +77,7 @@ func (s *Scanner) Scan() (tok token.Token, literal string, pos source.Pos) {
 		literal = s.scanIdentifier()
 		tok = token.Lookup(literal)
 		switch tok {
-		case token.Ident, token.Break, token.Continue, token.Return, token.True, token.False, token.Undefined:
+		case token.Ident, token.Break, token.Continue, token.Return, token.Export, token.True, token.False, token.Undefined:
 			insertSemi = true
 		}
 	case '0' <= ch && ch <= '9':

compiler/scanner/scanner_test.go

@@ -116,6 +116,7 @@ func TestScanner_Scan(t *testing.T) {
 		{token.Func, "func"},
 		{token.If, "if"},
 		{token.Return, "return"},
+		{token.Export, "export"},
 	}
 
 	// combine