Skip to content

Commit

Permalink
Generate Dataset code with meaningful fields names (#16386)
Browse files Browse the repository at this point in the history 
This PR is intended to help Logstash developers or users that want to better understand the code that's autogenerated to model a pipeline, assigning more meaningful names to the Datasets subclasses' fields.

Updates `FieldDefinition` to receive the name of the field from construction methods, so that it can be used during the code generation phase, instead of the existing incremental `field%n`.
Updates `ClassFields` to propagate the explicit field name down to the `FieldDefinitions`.
Update the `DatasetCompiler` that add fields to `ClassFields` to assign a proper name to generated Dataset's fields.
  • Loading branch information
@andsel
andsel authored Sep 4, 2024
1 parent 6e93b30 commit ac034a1
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 28 deletions.
8 changes: 8 additions & 0 deletions logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ public ValueSyntaxElement add(final Object obj) {
return addField(FieldDefinition.fromValue(definitions.size(), obj));
}

public ValueSyntaxElement add(final String fieldName, final Object obj) {
return addField(FieldDefinition.fromValue(fieldName, obj));
}

/**
* Adds a mutable field of the given type, that doesn't have a default value and is not
* initialized by a constructor assignment.
Expand All @@ -54,6 +58,10 @@ public ValueSyntaxElement add(final Class<?> type) {
return addField(FieldDefinition.mutableUnassigned(definitions.size(), type));
}

public ValueSyntaxElement add(final String fieldName, final Class<?> type) {
return addField(FieldDefinition.mutableUnassigned(fieldName, type));
}

/**
* Add a {@link Closure} that should be executed in the constructor after field assignments
* have been executed.
Expand Down
61 changes: 33 additions & 28 deletions logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.jruby.RubyArray;
import org.jruby.RubyHash;
Expand Down Expand Up @@ -58,8 +59,8 @@ public static ComputeStepSyntaxElement<SplitDataset> splitDataset(
final EventCondition condition)
{
final ClassFields fields = new ClassFields();
final ValueSyntaxElement ifData = fields.add(new ArrayList<>());
final ValueSyntaxElement elseData = fields.add(new ArrayList<>());
final ValueSyntaxElement ifData = fields.add("ifData", new ArrayList<>());
final ValueSyntaxElement elseData = fields.add("elseData", new ArrayList<>());
final ValueSyntaxElement right = fields.add(DatasetCompiler.Complement.class);
final VariableDefinition event =
new VariableDefinition(JrubyEventExtLibrary.RubyEvent.class, "event");
Expand All @@ -75,17 +76,16 @@ public static ComputeStepSyntaxElement<SplitDataset> splitDataset(
)
)
);
final ValueSyntaxElement conditionField = fields.add(condition);
final ValueSyntaxElement conditionField = fields.add("condition", condition);
final DatasetCompiler.ComputeAndClear compute;
if (parents.isEmpty()) {
compute = withOutputBuffering(
conditionalLoop(event, BATCH_ARG, conditionField, ifData, elseData),
Closure.wrap(clear(elseData)), ifData, fields
);
} else {
final Collection<ValueSyntaxElement> parentFields =
parents.stream().map(fields::add).collect(Collectors.toList());
final ValueSyntaxElement inputBuffer = fields.add(new ArrayList<>());
final Collection<ValueSyntaxElement> parentFields = createParentStatementsFields(parents, fields);
final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", new ArrayList<>());
compute = withOutputBuffering(
withInputBuffering(
conditionalLoop(event, inputBuffer, conditionField, ifData, elseData),
Expand All @@ -111,20 +111,17 @@ public static ComputeStepSyntaxElement<Dataset> filterDataset(
final AbstractFilterDelegatorExt plugin)
{
final ClassFields fields = new ClassFields();
final ValueSyntaxElement outputBuffer = fields.add(new ArrayList<>());
final ValueSyntaxElement outputBuffer = fields.add("outputBuffer", new ArrayList<>());
final Closure clear = Closure.wrap();
final Closure compute;
if (parents.isEmpty()) {
compute = filterBody(outputBuffer, BATCH_ARG, fields, plugin);
} else {
final Collection<ValueSyntaxElement> parentFields = parents
.stream()
.map(fields::add)
.collect(Collectors.toList()
);
final Collection<ValueSyntaxElement> parentFields = createParentStatementsFields(parents, fields);

@SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray();
clear.add(clearSyntax(parentFields));
final ValueSyntaxElement inputBufferField = fields.add(inputBuffer);
final ValueSyntaxElement inputBufferField = fields.add("inputBuffer", inputBuffer);
compute = withInputBuffering(
filterBody(outputBuffer, inputBufferField, fields, plugin),
parentFields, inputBufferField
Expand All @@ -133,6 +130,21 @@ public static ComputeStepSyntaxElement<Dataset> filterDataset(
return prepare(withOutputBuffering(compute, clear, outputBuffer, fields));
}

private static Collection<ValueSyntaxElement> createParentStatementsFields(Collection<Dataset> parents, ClassFields fields) {
if (parents.size() == 1) {
return List.of(fields.add("parentStatement", parents.iterator().next()));
}

final Collection<ValueSyntaxElement> parentFields = new ArrayList<>();
int i = 0;
for (Dataset parent : parents) {
ValueSyntaxElement add = fields.add("parentStatement" + i, parent);
parentFields.add(add);
i++;
}
return parentFields;
}

/**
* <p>Builds a terminal {@link Dataset} for the filters from the given parent {@link Dataset}s.</p>
* <p>If the given set of parent {@link Dataset} is empty the sum is defined as the
Expand All @@ -155,10 +167,7 @@ public static Dataset terminalFilterDataset(final Collection<Dataset> parents) {
}

final ClassFields fields = new ClassFields();
final Collection<ValueSyntaxElement> parentFields = parents
.stream()
.map(fields::add)
.collect(Collectors.toList());
final Collection<ValueSyntaxElement> parentFields = createParentStatementsFields(parents, fields);
@SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray();
final ValueSyntaxElement inputBufferField = fields.add(inputBuffer);
final ValueSyntaxElement outputBufferField = fields.add(new ArrayList<>());
Expand Down Expand Up @@ -199,10 +208,7 @@ public static Dataset terminalOutputDataset(final Collection<Dataset> parents) {
}

final ClassFields fields = new ClassFields();
final Collection<ValueSyntaxElement> parentFields = parents
.stream()
.map(fields::add)
.collect(Collectors.toList());
final Collection<ValueSyntaxElement> parentFields = createParentStatementsFields(parents, fields);
final Closure compute = Closure.wrap(parentFields
.stream()
.map(DatasetCompiler::computeDataset)
Expand Down Expand Up @@ -236,16 +242,15 @@ public static ComputeStepSyntaxElement<Dataset> outputDataset(
final ClassFields fields = new ClassFields();
final Closure clearSyntax;
final Closure computeSyntax;
final ValueSyntaxElement outputField = fields.add(output);
final ValueSyntaxElement outputField = fields.add("outputDelegator", output);
if (parents.isEmpty()) {
clearSyntax = Closure.EMPTY;
computeSyntax = Closure.wrap(
setPluginIdForLog4j(outputField),
invokeOutput(outputField, BATCH_ARG),
unsetPluginIdForLog4j());
} else {
final Collection<ValueSyntaxElement> parentFields =
parents.stream().map(fields::add).collect(Collectors.toList());
final Collection<ValueSyntaxElement> parentFields = createParentStatementsFields(parents, fields);
@SuppressWarnings("rawtypes")
final RubyArray buffer = RubyUtil.RUBY.newArray();
final Closure inlineClear;
Expand All @@ -256,7 +261,7 @@ public static ComputeStepSyntaxElement<Dataset> outputDataset(
inlineClear = Closure.EMPTY;
clearSyntax = clearSyntax(parentFields);
}
final ValueSyntaxElement inputBuffer = fields.add(buffer);
final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", buffer);
computeSyntax = withInputBuffering(
Closure.wrap(
setPluginIdForLog4j(outputField),
Expand All @@ -283,7 +288,7 @@ private static Closure filterBody(
final ClassFields fields,
final AbstractFilterDelegatorExt plugin)
{
final ValueSyntaxElement filterField = fields.add(plugin);
final ValueSyntaxElement filterField = fields.add("plugin", plugin);
final Closure body = Closure.wrap(
setPluginIdForLog4j(filterField),
buffer(outputBuffer, filterField.call("multiFilter", inputBuffer))
Expand Down Expand Up @@ -378,14 +383,14 @@ private static MethodLevelSyntaxElement callFilterFlush(final ClassFields fields
final boolean shutdownOnly) {
final MethodLevelSyntaxElement condition;
final ValueSyntaxElement flushArgs;
final ValueSyntaxElement flushFinal = fields.add(flushOpts(true));
final ValueSyntaxElement flushFinal = fields.add("shutdownFlushOptions", flushOpts(true));
if (shutdownOnly) {
condition = SyntaxFactory.and(FLUSH_ARG, SHUTDOWN_ARG);
flushArgs = flushFinal;
} else {
condition = FLUSH_ARG;
flushArgs = SyntaxFactory.ternary(
SHUTDOWN_ARG, flushFinal, fields.add(flushOpts(false))
SHUTDOWN_ARG, flushFinal, fields.add("flushOptions", flushOpts(false))
);
}
return SyntaxFactory.ifCondition(
Expand Down
17 changes: 17 additions & 0 deletions logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ public static FieldDefinition fromValue(final int index, final Object value) {
);
}

public static FieldDefinition fromValue(final String fieldName, final Object value) {
return new FieldDefinition(
variableDefinition(value.getClass(), fieldName), false, null, value
);
}

/**
* Creates a mutable field with given type and without an assigned value.
* @param index Index for naming
Expand All @@ -58,6 +64,12 @@ public static FieldDefinition mutableUnassigned(final int index, final Class<?>
);
}

public static FieldDefinition mutableUnassigned(final String fieldName, final Class<?> type) {
return new FieldDefinition(
variableDefinition(type, fieldName), true, null, null
);
}

private FieldDefinition(final VariableDefinition typeDef, final boolean mutable,
final SyntaxElement initializer, final Object ctorArgument) {
this.def = typeDef;
Expand Down Expand Up @@ -98,4 +110,9 @@ public String generateCode() {
private static VariableDefinition variableDefinition(final Class<?> type, final int index) {
return new VariableDefinition(type, String.format("field%d", index));
}

private static VariableDefinition variableDefinition(final Class<?> type, final String fieldName) {
return new VariableDefinition(type, String.format("%sField", fieldName));
}

}

0 comments on commit ac034a1

Please sign in to comment.