diff --git a/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java new file mode 100644 index 000000000..6f319a23e --- /dev/null +++ b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/DSLSyntaxException.java @@ -0,0 +1,8 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions; + + +public class DSLSyntaxException extends Exception { + public DSLSyntaxException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java new file mode 100644 index 000000000..7bd8f1562 --- /dev/null +++ b/backend/src/main/java/com/griddynamics/genesis/template/dsl/groovy/transformations/exceptions/MacroParametersException.java @@ -0,0 +1,8 @@ +package com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions; + + +public class MacroParametersException extends DSLSyntaxException { + public MacroParametersException(String message) { + super(message); + } +} diff --git a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala index ad6db89f7..fe9a5c4f8 100755 --- a/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/service/impl/GroovyTemplateService.scala @@ -48,6 +48,7 @@ import org.codehaus.groovy.control.CompilerConfiguration import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer import support.VariablesSupport import com.griddynamics.genesis.template.dsl.groovy.transformations.{MacroExpand, PhaseContainer, Context} +import scala.tools.ant.sabbus.CompilationFailure @RemoteGateway("groovy template service") class GroovyTemplateService(val templateRepoService : TemplateRepoService, @@ -156,7 +157,8 @@ class GroovyTemplateService(val templateRepoService : TemplateRepoService, groovyShell.evaluate(body) projectId.foreach (evaluateIncludes(_, templateDecl.includes, groovyShell)) } catch { - case e: GroovyRuntimeException => throw new IllegalStateException("can't process template", e) + case compilation: CompilationFailure => throw new IllegalStateException("Conpilation error: " + compilation.message, compilation) + case e: GroovyRuntimeException => throw new IllegalStateException("can't process template: " + e.getMessage, e) } templateDecl.bodies.headOption.map { body => DslDelegate(body).to(builder).newTemplate() } } diff --git a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala index ddcf783ff..64170bac7 100644 --- a/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala +++ b/backend/src/main/scala/com/griddynamics/genesis/template/dsl/groovy/transformations/MacroASTTransformation.scala @@ -3,12 +3,14 @@ package com.griddynamics.genesis.template.dsl.groovy.transformations import org.codehaus.groovy.transform.{ASTTransformation, GroovyASTTransformation} import org.codehaus.groovy.control.{SourceUnit, CompilePhase} import com.griddynamics.genesis.util.Logging -import org.codehaus.groovy.ast.{CodeVisitorSupport, ASTNode} +import org.codehaus.groovy.ast.{Parameter, CodeVisitorSupport, ASTNode} import org.codehaus.groovy.ast.expr._ import scala.collection.mutable import org.codehaus.groovy.ast.stmt.{ExpressionStatement, Statement, BlockStatement} import java.util import scala.collection.JavaConversions._ +import com.griddynamics.genesis.template.dsl.groovy.transformations.exceptions.MacroParametersException +import org.codehaus.groovy.control.messages.SimpleMessage @GroovyASTTransformation(phase = CompilePhase.CONVERSION) class MacroASTTransformation extends ASTTransformation with Logging { @@ -16,21 +18,171 @@ class MacroASTTransformation extends ASTTransformation with Logging { log.trace(s"Inside a macros transformer: ${nodes.length}") val methods = source.getAST.getStatementBlock.getStatements val it = methods.iterator().next() - val collector = new MacroCollector + val collector = new MacroCollector(source) it.visit(new NameMatchArgumentTransformer("template", collector)) if (collector.macros.size > 0) { - methods.iterator().next().visit(new MacroExpandVisitor(collector.macros.toMap)) + val replacementMap = new mutable.HashMap[Statement, Expression]() + log.trace(s"Found ${collector.macros.size} macros") + methods.iterator().next().visit(new MacroExpandVisitor(collector.macros.toMap, replacementMap, source)) } + log.trace("Done") } } -class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends CodeVisitorSupport with Logging { +trait SourceReporting { + def source: SourceUnit + def error[B](message: String)(value: B) = { + source.getErrorCollector.addError(new SimpleMessage(message, "", source)) + value + } +} + +class ApplyVariablesVisitor(values: Map[String, Expression], replacements: mutable.Map[Statement, Expression]) extends CodeVisitorSupport with ExpressionTransformer + with Logging { + override def visitExpressionStatement(statement: ExpressionStatement) { + val expression: Expression = statement.getExpression + //expression.visit(this) + val toChange: Expression = expression + val newExpression: Expression = toChange.transformExpression(this) + statement.setExpression(newExpression) + super.visitExpressionStatement(statement) + } + + + override def visitMapEntryExpression(expression: MapEntryExpression) { + val key: Expression = expression.getKeyExpression + val value: Expression = expression.getValueExpression + expression.setKeyExpression(transform(key)) + expression.setValueExpression(transform(value)) + super.visitMapEntryExpression(expression) + } + + override def visitBinaryExpression(expression: BinaryExpression) { + expression.setLeftExpression(transform(expression.getLeftExpression)) + expression.setRightExpression(transform(expression.getRightExpression)) + super.visitBinaryExpression(expression) + } + + def transform(expression: Expression): Expression = { + expression match { + case variable: VariableExpression => { + values.get(variable.getName).getOrElse(variable) + } + case constant: ConstantExpression if constant.getText.startsWith("$") => + values.get(constant.getText).getOrElse(constant) + case other => other + } + } +} +class MacroExpandVisitor(val macrodefs: Map[String, Macro], replacements: mutable.Map[Statement,Expression], val source: SourceUnit) extends CodeVisitorSupport with Logging with SourceReporting { val nameExtract: PartialFunction[Expression, String] = { case call if call.isInstanceOf[MethodCallExpression] => call.asInstanceOf[MethodCallExpression].getMethodAsString } + private def substitute(code: Macro, call: MethodCallExpression): Statement = { + val arguments: Expression = call.getArguments + //possible variants: ArgumentListExpression, Map + val passed: Seq[PassedParameter] = arguments match { + case list: ArgumentListExpression => list.getExpressions.zipWithIndex.map({case (expr, index) => new PositionedParameter(index, expr)}).toSeq + case tuple: TupleExpression => tuple.getExpressions.flatMap(e => e match { + case named: NamedArgumentListExpression => { + named.getMapEntryExpressions.map(expr => expr.getKeyExpression match { + case constant: ConstantExpression => Some(NamedParameter(constant.getText, expr.getValueExpression)) + case other => error("Only constants allowed in macro named arguments")(None) + }) + } + }).flatten + case other => error("Macro parameters can be passed as comma-separated list or as named pairs")(Seq()) + } + //apply + val values: Map[String, Option[Expression]] = code.parameters.zipWithIndex.map({case ((name, value), idx) => { + val found = passed.find({ + case named: NamedParameter => named.name == name + case pos: PositionedParameter => pos.position == idx + }).map(_.value).orElse(value) + (name, found) + }}).toMap + val unsassigned = values.collect({case (name, value) if value.isEmpty || value.get == null => name}) + if (unsassigned.size > 0) { + error(s"Some arguments for macro `${code.name}` were not assigned and they don't have default value: " + unsassigned.mkString(", "))(code.code) + } else { + applyArguments(values.map({case (name, expr) => (name, expr.get)}), code.code) + } + } + + private def applyArguments(values: Map[String, Expression], code: BlockStatement) : BlockStatement = { + val visitor = new ApplyVariablesVisitor(values, replacements) + log.debug(s"Applying variables $values") + val copy = copyBlock(code) + log.trace(s"Initial code is: $code") + copy.visit(visitor) + log.debug(s"Copy with expanded variables: $copy") + copy + } - def replaceStatement(statement: BlockStatement, expression: ExpressionStatement): Statement = { + private def copyBlock(original: BlockStatement): BlockStatement = { + val bs = new BlockStatement() + addBlockToBlock(original, bs) + bs + } + + private def addBlockToBlock(source: BlockStatement, target: BlockStatement) { + source.getStatements.foreach(addStatement(target)) + } + + private val copy: PartialFunction[Expression, Expression] = { + case variable: VariableExpression => new VariableExpression(variable.getName) + case constant: ConstantExpression => new ConstantExpression(constant.getText) + case binary: BinaryExpression => new BinaryExpression(copy(binary.getLeftExpression), binary.getOperation, + copy(binary.getRightExpression)) + case closure: ClosureExpression => copyClosureExpression(closure) + case arguments: ArgumentListExpression => new ArgumentListExpression(arguments.getExpressions.map(copy)) + case mapEntry: MapEntryExpression => new MapEntryExpression(copy(mapEntry.getKeyExpression), copy(mapEntry.getValueExpression)) + } + + private def copyClosureExpression(closure: ClosureExpression): ClosureExpression = { + def copyParameters (closure: ClosureExpression) = { + closure.getParameters.map(p => new Parameter(p.getType, p.getName, copy(p.getInitialExpression))) + } + closure.getCode match { + case bs: BlockStatement => new ClosureExpression(copyParameters(closure), copyBlock(bs)) + } + } + + private def addBinaryExpression(be: BinaryExpression, bs: BlockStatement) = { + bs.addStatement(new ExpressionStatement(copy(be))) + } + + private def addMethodCall(objectExpr: Expression, method: String, arguments: Expression, statement: BlockStatement) { + val newExpr = new MethodCallExpression(objectExpr, method, copy(arguments)) + statement.addStatement(new ExpressionStatement(newExpr)) + } + + private def addMapExpression(map: MapExpression, statement: BlockStatement) { + val copies = map.getMapEntryExpressions.map(entry => { + copy(entry).asInstanceOf[MapEntryExpression] + }) + val newExpr = new MapExpression(copies.toList) + statement.addStatement(new ExpressionStatement(newExpr)) + } + + private def addStatement(blockStatement: BlockStatement)(statement: Statement) { + statement match { + case bs: BlockStatement => addBlockToBlock(bs, blockStatement) + case es: ExpressionStatement => { + val expr: Expression = es.getExpression + expr match { + case constant: ConstantExpression => blockStatement.addStatement(es) + case de: DeclarationExpression => blockStatement.addStatement(es) + case be: BinaryExpression => addBinaryExpression(be, blockStatement) + case method: MethodCallExpression => addMethodCall(method.getObjectExpression, method.getMethodAsString, method.getArguments, blockStatement) + case map: MapExpression => addMapExpression(map, blockStatement) + } + } + } + } + + private def replaceStatement(statement: BlockStatement, expression: ExpressionStatement): Statement = { val initialStatements: util.List[Statement] = statement.getStatements val position = initialStatements.indexOf(expression) expression.getExpression match { @@ -38,10 +190,12 @@ class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends val key: String = nameExtract(expression.getExpression) log.trace(s"Expanding call of $key at position $position") macrodefs.get(key).map(e => { - initialStatements.set(position, e.getCode) - }) orElse(throw new IllegalArgumentException(s"Macro $key not found anywhere in template")) + val code: Statement = if (e.parameters.isEmpty) e.code else substitute(e, mce) + initialStatements.set(position, code) + code + }) orElse error(s"Macro $key not found anywhere in template")(None) case other => - throw new IllegalArgumentException("Macro call must have a form of function call: " + expression.getExpression) + error("Macro call must have a form of function call: " + expression.getExpression)_ } val cleared = initialStatements.filterNot(_.getStatementLabel == "macro") new BlockStatement(cleared, statement.getVariableScope) @@ -84,8 +238,23 @@ class MacroExpandVisitor(val macrodefs: Map[String, ClosureExpression]) extends } } -class MacroCollector extends ExpressionTransformer with Logging { - var macros: mutable.Map[String, ClosureExpression] = new mutable.HashMap[String, ClosureExpression]() +case class Macro(name: String, parameters: List[Macro.ParameterDefinition], code: BlockStatement) { + val parametersMap = parameters.toMap + val zippedMap = parameters.zipWithIndex.map({case (definition,index) => (index,definition)}).toMap +} + +sealed trait PassedParameter { + def value: Expression +} +case class NamedParameter(name: String, value: Expression) extends PassedParameter +case class PositionedParameter(position: Int, value: Expression) extends PassedParameter + +object Macro { + type ParameterDefinition = (String, Option[Expression]) +} + +class MacroCollector(val source: SourceUnit) extends ExpressionTransformer with Logging with SourceReporting { + var macros: mutable.Map[String, Macro] = new mutable.HashMap[String, Macro]() def transform(expression: Expression) : Expression = { expression match { case a: ClosureExpression => { @@ -121,19 +290,33 @@ class MacroCollector extends ExpressionTransformer with Logging { } def saveExpr(expression: MethodCallExpression) { - def parseArgurmentList(mapExpr: NamedArgumentListExpression): (String, ClosureExpression) = { + def parseArgumentList(mapExpr: NamedArgumentListExpression): Option[(String, ClosureExpression)] = { val top = mapExpr.getMapEntryExpressions.iterator().next() val key: String = top.getKeyExpression.asInstanceOf[ConstantExpression].getValue.toString - val closure = top.getValueExpression.asInstanceOf[ClosureExpression] - (key, closure) + top.getValueExpression match { + case closure: ClosureExpression => Some(key, closure) + case other => error(s"Macro body must be a closure: $key; actual $other")(None) + } + } + def parseMacro(name: String, closure: ClosureExpression): Option[Macro] = { + val parameters = closure.getParameters.map(parameter => { + new Macro.ParameterDefinition(parameter.getName, Some(parameter.getInitialExpression)) + }) + closure.getCode match { + case bs: BlockStatement => Some(Macro(name, parameters.toList, closure.getCode.asInstanceOf[BlockStatement])) + case other => error(s"Macro body must be a block statement: $name; actual: $other")(None) + } + } - val (name, body) = expression.getArguments match { + val nameAndCode: Option[(String, ClosureExpression)] = expression.getArguments match { case mapExpr: NamedArgumentListExpression => - parseArgurmentList(mapExpr) + parseArgumentList(mapExpr) case tuple: TupleExpression => - parseArgurmentList(tuple.getExpression(0).asInstanceOf[NamedArgumentListExpression]) + tuple.getExpression(0) match { + case arguments: NamedArgumentListExpression => parseArgumentList(arguments) + } } - macros(name) = body + nameAndCode.flatMap ({case (name,body) => parseMacro(name,body) }) map (result => macros(result.name) = result) } def evalStatement(statement: Statement): Boolean = { diff --git a/backend/src/test/resources/groovy/Macros.genesis b/backend/src/test/resources/groovy/Macros.genesis index cefd07fbc..788ea97d6 100644 --- a/backend/src/test/resources/groovy/Macros.genesis +++ b/backend/src/test/resources/groovy/Macros.genesis @@ -5,13 +5,36 @@ template { createWorkflow("create") destroyWorkflow("destroy") + def MY_CONSTANT = "Set from constant" - defmacro "create_steps": { + defmacro "create_steps": { $message = "default" -> + 1 + 2 teststep { - text = "test from macro" + text = $message } } + defmacro "simple": { + teststep { + text = MY_CONSTANT + } + } + + defmacro "map": { $key, $mapValue -> + withMap { + text = "Test" + map = { [ $key: $mapValue ] } + } + } + + defmacro "bad_require": { $message -> + require { + $message { 1 == 2 } + "Oops again" { 2 == 1 } + } + + } + workflow("create") { steps { teststep { @@ -23,12 +46,22 @@ template { } } + workflow("maps") { + steps { + macro:map("operation", "subst") + } + } + workflow("macros") { + macro:bad_require("Oops") steps { teststep { - text = "It was here" + text = "Static" } + macro:create_steps("Passed from macro call") + macro:create_steps($message: "Set with map") macro:create_steps() + macro:simple() } } diff --git a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala index 5e14a2631..617ec646f 100644 --- a/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala +++ b/backend/src/test/scala/com/griddynamics/genesis/service/impl/MacrosTest.scala @@ -13,12 +13,18 @@ import com.griddynamics.genesis.template.support.DatabagDataSourceFactory import com.griddynamics.genesis.cache.NullCacheManager import com.griddynamics.genesis.template.VersionedTemplate import com.griddynamics.genesis.api -import com.griddynamics.genesis.plugin.StepBuilder +import com.griddynamics.genesis.plugin.{StepBuilderFactory, StepBuilder} +import com.griddynamics.genesis.service.TemplateDefinition +import com.griddynamics.genesis.api.Failure +import com.griddynamics.genesis.workflow.Step +import scala.beans.BeanProperty +import scala.collection.mutable +import scala.collection.JavaConversions._ class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniverse with Logging { val templateService = new GroovyTemplateService(templateRepoService, - List(new DoNothingStepBuilderFactory), new DefaultConversionService, + List(new DoNothingStepBuilderFactory, new StepWithMapFactory), new DefaultConversionService, Seq(new ListVarDSFactory, new DependentListVarDSFactory, new DatabagDataSourceFactory(databagRepository)), databagRepository, configService, NullCacheManager) @@ -28,13 +34,55 @@ class MacrosTest extends AssertionsForJUnit with MockitoSugar with DSLTestUniver when(configService.get(Matchers.any(), Matchers.any())).thenReturn(Some(new api.Configuration(Some(0), "", 0, None, Map()))) @Test - def testPhaseApplied() { - val createWorkflow = templateService.findTemplate(0, "Macros", "0.1", 0).flatMap(_.getWorkflow("macros")).get - val steps = createWorkflow.embody(Map()) - expectResult(2)(steps.regular.size) + def testStepsInserted() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + val workflow = template.flatMap(_.getWorkflow("macros")).get + val steps = workflow.embody(Map()) + expectResult(5)(steps.regular.size) val initialPhase: Option[StepBuilder] = steps.regular.find(_.phase == "auto_0") + val secondPhase: Option[StepBuilder] = steps.regular.find(_.phase == "auto_1") assert(initialPhase.isDefined) assert(initialPhase.get.getPrecedingPhases.isEmpty) + assert(secondPhase.isDefined) + assert(secondPhase.get.getPrecedingPhases.contains("auto_0")) + steps.regular.zip(Seq("Static", "Passed from macro call", + "Set with map", "default", "Set from constant")).map({ + case (step, message) => step.newStep.actualStep match { + case nothing: DoNothingStep => expectResult(message)(nothing.name) + } + }) + } + + @Test + def testFullRequireFromMacro() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + assert(template.isDefined) + val result = template.map(_.getValidWorkflow("macros")).get + expectResult(Failure(compoundServiceErrors = Seq("Oops", "Oops again")))(result) + } + + @Test + def testMacroWithMap() { + val template: Option[TemplateDefinition] = templateService.findTemplate(0, "Macros", "0.1", 0) + val result = template.get.getWorkflow("maps").get + val steps = result.embody(Map()) + val values: Map[String, String] = steps.regular(0).newStep.actualStep.asInstanceOf[StepWithMap].values + expectResult(Map("operation" -> "subst"))(values) + } +} + +case class StepWithMap(name: String, values: Map[String,String]) extends Step { + override def stepDescription = "Best step ever!" +} + +class StepWithMapFactory extends StepBuilderFactory { + val stepName = "withMap" + + def newStepBuilder = new StepBuilder { + @BeanProperty var text: String = _ + @BeanProperty var map: java.util.Map[String, String] = _ + + def getDetails = StepWithMap(text, map.toMap) } }