From ac034a14ee422148483d42a51b68f07d1a38494c Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 4 Sep 2024 11:10:29 +0200 Subject: [PATCH] Generate Dataset code with meaningful fields names (#16386) This PR is intended to help Logstash developers or users that want to better understand the code that's autogenerated to model a pipeline, assigning more meaningful names to the Datasets subclasses' fields. Updates `FieldDefinition` to receive the name of the field from construction methods, so that it can be used during the code generation phase, instead of the existing incremental `field%n`. Updates `ClassFields` to propagate the explicit field name down to the `FieldDefinitions`. Update the `DatasetCompiler` that add fields to `ClassFields` to assign a proper name to generated Dataset's fields. --- .../config/ir/compiler/ClassFields.java | 8 +++ .../config/ir/compiler/DatasetCompiler.java | 61 ++++++++++--------- .../config/ir/compiler/FieldDefinition.java | 17 ++++++ 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java index b34131bcd31..b4f5cf5e041 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java @@ -43,6 +43,10 @@ public ValueSyntaxElement add(final Object obj) { return addField(FieldDefinition.fromValue(definitions.size(), obj)); } + public ValueSyntaxElement add(final String fieldName, final Object obj) { + return addField(FieldDefinition.fromValue(fieldName, obj)); + } + /** * Adds a mutable field of the given type, that doesn't have a default value and is not * initialized by a constructor assignment. @@ -54,6 +58,10 @@ public ValueSyntaxElement add(final Class type) { return addField(FieldDefinition.mutableUnassigned(definitions.size(), type)); } + public ValueSyntaxElement add(final String fieldName, final Class type) { + return addField(FieldDefinition.mutableUnassigned(fieldName, type)); + } + /** * Add a {@link Closure} that should be executed in the constructor after field assignments * have been executed. diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java index 689fb00302d..e3db9a0105d 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; import org.jruby.RubyArray; import org.jruby.RubyHash; @@ -58,8 +59,8 @@ public static ComputeStepSyntaxElement splitDataset( final EventCondition condition) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement ifData = fields.add(new ArrayList<>()); - final ValueSyntaxElement elseData = fields.add(new ArrayList<>()); + final ValueSyntaxElement ifData = fields.add("ifData", new ArrayList<>()); + final ValueSyntaxElement elseData = fields.add("elseData", new ArrayList<>()); final ValueSyntaxElement right = fields.add(DatasetCompiler.Complement.class); final VariableDefinition event = new VariableDefinition(JrubyEventExtLibrary.RubyEvent.class, "event"); @@ -75,7 +76,7 @@ public static ComputeStepSyntaxElement splitDataset( ) ) ); - final ValueSyntaxElement conditionField = fields.add(condition); + final ValueSyntaxElement conditionField = fields.add("condition", condition); final DatasetCompiler.ComputeAndClear compute; if (parents.isEmpty()) { compute = withOutputBuffering( @@ -83,9 +84,8 @@ public static ComputeStepSyntaxElement splitDataset( Closure.wrap(clear(elseData)), ifData, fields ); } else { - final Collection parentFields = - parents.stream().map(fields::add).collect(Collectors.toList()); - final ValueSyntaxElement inputBuffer = fields.add(new ArrayList<>()); + final Collection parentFields = createParentStatementsFields(parents, fields); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", new ArrayList<>()); compute = withOutputBuffering( withInputBuffering( conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), @@ -111,20 +111,17 @@ public static ComputeStepSyntaxElement filterDataset( final AbstractFilterDelegatorExt plugin) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement outputBuffer = fields.add(new ArrayList<>()); + final ValueSyntaxElement outputBuffer = fields.add("outputBuffer", new ArrayList<>()); final Closure clear = Closure.wrap(); final Closure compute; if (parents.isEmpty()) { compute = filterBody(outputBuffer, BATCH_ARG, fields, plugin); } else { - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList() - ); + final Collection parentFields = createParentStatementsFields(parents, fields); + @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); clear.add(clearSyntax(parentFields)); - final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); + final ValueSyntaxElement inputBufferField = fields.add("inputBuffer", inputBuffer); compute = withInputBuffering( filterBody(outputBuffer, inputBufferField, fields, plugin), parentFields, inputBufferField @@ -133,6 +130,21 @@ public static ComputeStepSyntaxElement filterDataset( return prepare(withOutputBuffering(compute, clear, outputBuffer, fields)); } + private static Collection createParentStatementsFields(Collection parents, ClassFields fields) { + if (parents.size() == 1) { + return List.of(fields.add("parentStatement", parents.iterator().next())); + } + + final Collection parentFields = new ArrayList<>(); + int i = 0; + for (Dataset parent : parents) { + ValueSyntaxElement add = fields.add("parentStatement" + i, parent); + parentFields.add(add); + i++; + } + return parentFields; + } + /** *

Builds a terminal {@link Dataset} for the filters from the given parent {@link Dataset}s.

*

If the given set of parent {@link Dataset} is empty the sum is defined as the @@ -155,10 +167,7 @@ public static Dataset terminalFilterDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); final ValueSyntaxElement outputBufferField = fields.add(new ArrayList<>()); @@ -199,10 +208,7 @@ public static Dataset terminalOutputDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); final Closure compute = Closure.wrap(parentFields .stream() .map(DatasetCompiler::computeDataset) @@ -236,7 +242,7 @@ public static ComputeStepSyntaxElement outputDataset( final ClassFields fields = new ClassFields(); final Closure clearSyntax; final Closure computeSyntax; - final ValueSyntaxElement outputField = fields.add(output); + final ValueSyntaxElement outputField = fields.add("outputDelegator", output); if (parents.isEmpty()) { clearSyntax = Closure.EMPTY; computeSyntax = Closure.wrap( @@ -244,8 +250,7 @@ public static ComputeStepSyntaxElement outputDataset( invokeOutput(outputField, BATCH_ARG), unsetPluginIdForLog4j()); } else { - final Collection parentFields = - parents.stream().map(fields::add).collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray buffer = RubyUtil.RUBY.newArray(); final Closure inlineClear; @@ -256,7 +261,7 @@ public static ComputeStepSyntaxElement outputDataset( inlineClear = Closure.EMPTY; clearSyntax = clearSyntax(parentFields); } - final ValueSyntaxElement inputBuffer = fields.add(buffer); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", buffer); computeSyntax = withInputBuffering( Closure.wrap( setPluginIdForLog4j(outputField), @@ -283,7 +288,7 @@ private static Closure filterBody( final ClassFields fields, final AbstractFilterDelegatorExt plugin) { - final ValueSyntaxElement filterField = fields.add(plugin); + final ValueSyntaxElement filterField = fields.add("plugin", plugin); final Closure body = Closure.wrap( setPluginIdForLog4j(filterField), buffer(outputBuffer, filterField.call("multiFilter", inputBuffer)) @@ -378,14 +383,14 @@ private static MethodLevelSyntaxElement callFilterFlush(final ClassFields fields final boolean shutdownOnly) { final MethodLevelSyntaxElement condition; final ValueSyntaxElement flushArgs; - final ValueSyntaxElement flushFinal = fields.add(flushOpts(true)); + final ValueSyntaxElement flushFinal = fields.add("shutdownFlushOptions", flushOpts(true)); if (shutdownOnly) { condition = SyntaxFactory.and(FLUSH_ARG, SHUTDOWN_ARG); flushArgs = flushFinal; } else { condition = FLUSH_ARG; flushArgs = SyntaxFactory.ternary( - SHUTDOWN_ARG, flushFinal, fields.add(flushOpts(false)) + SHUTDOWN_ARG, flushFinal, fields.add("flushOptions", flushOpts(false)) ); } return SyntaxFactory.ifCondition( diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java index b3e481e6fd4..4ed2ba3b20e 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java @@ -46,6 +46,12 @@ public static FieldDefinition fromValue(final int index, final Object value) { ); } + public static FieldDefinition fromValue(final String fieldName, final Object value) { + return new FieldDefinition( + variableDefinition(value.getClass(), fieldName), false, null, value + ); + } + /** * Creates a mutable field with given type and without an assigned value. * @param index Index for naming @@ -58,6 +64,12 @@ public static FieldDefinition mutableUnassigned(final int index, final Class ); } + public static FieldDefinition mutableUnassigned(final String fieldName, final Class type) { + return new FieldDefinition( + variableDefinition(type, fieldName), true, null, null + ); + } + private FieldDefinition(final VariableDefinition typeDef, final boolean mutable, final SyntaxElement initializer, final Object ctorArgument) { this.def = typeDef; @@ -98,4 +110,9 @@ public String generateCode() { private static VariableDefinition variableDefinition(final Class type, final int index) { return new VariableDefinition(type, String.format("field%d", index)); } + + private static VariableDefinition variableDefinition(final Class type, final String fieldName) { + return new VariableDefinition(type, String.format("%sField", fieldName)); + } + }