From 24f6f65220f6cf7f12d523d10b3ed59d8f6dc76d Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 11:05:23 -0600 Subject: [PATCH 01/44] Fix ExactMatch Filter for Non-Convertible Types --- .../deephaven/engine/table/impl/select/MatchFilter.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 2144b522bb1..5446022278e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -497,8 +497,13 @@ Object convertStringLiteral(String str) { } }; } - throw new IllegalArgumentException( - "Unknown type " + cls.getName() + " for MatchFilter value auto-conversion"); + return new ColumnTypeConvertor() { + @Override + Object convertStringLiteral(String str) { + throw new IllegalArgumentException( + "Can't convert String to " + cls.getName() + " for MatchFilter value auto-conversion"); + } + }; } } From 1bec5c331c8249826735aba908484cf28315d94e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 11:20:41 -0600 Subject: [PATCH 02/44] Add LocalDate, LocalTime, LocalDateTime, ZoneDateTime Converters --- .../engine/table/impl/select/MatchFilter.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 5446022278e..c5a24c5bc5c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -28,6 +28,10 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Stream; @@ -459,6 +463,54 @@ Object convertStringLiteral(String str) { } }; } + if (cls == LocalDate.class) { + return new ColumnTypeConvertor() { + @Override + Object convertStringLiteral(String str) { + if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { + throw new IllegalArgumentException( + "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + } + return LocalDate.parse(str.substring(1, str.length() - 1)); + } + }; + } + if (cls == LocalTime.class) { + return new ColumnTypeConvertor() { + @Override + Object convertStringLiteral(String str) { + if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { + throw new IllegalArgumentException( + "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + } + return LocalTime.parse(str.substring(1, str.length() - 1)); + } + }; + } + if (cls == LocalDateTime.class) { + return new ColumnTypeConvertor() { + @Override + Object convertStringLiteral(String str) { + if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { + throw new IllegalArgumentException( + "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + } + return LocalDateTime.parse(str.substring(1, str.length() - 1)); + } + }; + } + if (cls == ZonedDateTime.class) { + return new ColumnTypeConvertor() { + @Override + Object convertStringLiteral(String str) { + if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { + throw new IllegalArgumentException( + "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + } + return ZonedDateTime.parse(str.substring(1, str.length() - 1)); + } + }; + } if (cls == Object.class) { return new ColumnTypeConvertor() { @Override From 89eee34c1cdd2f58869b574f77b14cf344da5e28 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 13:41:27 -0600 Subject: [PATCH 03/44] Fix #5526 by Sharing Conversion With MatchFilter --- .../table/impl/select/ByteRangeFilter.java | 10 +- .../table/impl/select/CharRangeFilter.java | 10 +- .../table/impl/select/DoubleRangeFilter.java | 10 +- .../table/impl/select/FloatRangeFilter.java | 10 +- .../table/impl/select/IntRangeFilter.java | 10 +- .../table/impl/select/LongRangeFilter.java | 10 +- .../engine/table/impl/select/MatchFilter.java | 79 ++++++++------ .../impl/select/RangeConditionFilter.java | 103 ++++++++---------- .../table/impl/select/ShortRangeFilter.java | 10 +- .../table/impl/select/WhereFilterFactory.java | 53 ++++++--- 10 files changed, 163 insertions(+), 142 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index d506eee8bc5..5547aa1633a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -51,16 +51,16 @@ public ByteRangeFilter(String columnName, byte val1, byte val2, boolean lowerInc } } - static WhereFilter makeByteRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeByteRangeFilter(String columnName, Condition condition, byte value) { switch (condition) { case LESS_THAN: - return lt(columnName, RangeConditionFilter.parseByteFilter(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, RangeConditionFilter.parseByteFilter(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, RangeConditionFilter.parseByteFilter(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, RangeConditionFilter.parseByteFilter(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index f5d640d13f5..0041a195cf8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -47,16 +47,16 @@ public CharRangeFilter(String columnName, char val1, char val2, boolean lowerInc } } - static WhereFilter makeCharRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeCharRangeFilter(String columnName, Condition condition, char value) { switch (condition) { case LESS_THAN: - return lt(columnName, RangeConditionFilter.parseCharFilter(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, RangeConditionFilter.parseCharFilter(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, RangeConditionFilter.parseCharFilter(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, RangeConditionFilter.parseCharFilter(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index 1b195e01b0b..bfcaae139b6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -63,16 +63,16 @@ public static WhereFilter makeRange(String columnName, String val) { (double) (positiveOrZero ? parsed + offset : parsed - offset), positiveOrZero, !positiveOrZero); } - static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, double value) { switch (condition) { case LESS_THAN: - return lt(columnName, Double.parseDouble(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, Double.parseDouble(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, Double.parseDouble(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, Double.parseDouble(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index 2cdebf0bde7..9fe1d3d2e98 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -59,16 +59,16 @@ public static WhereFilter makeRange(String columnName, String val) { (float) (positiveOrZero ? parsed + offset : parsed - offset), positiveOrZero, !positiveOrZero); } - static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, float value) { switch (condition) { case LESS_THAN: - return lt(columnName, Float.parseFloat(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, Float.parseFloat(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, Float.parseFloat(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, Float.parseFloat(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index ee4a345e2a2..4f12b52867c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -51,16 +51,16 @@ public IntRangeFilter(String columnName, int val1, int val2, boolean lowerInclus } } - static WhereFilter makeIntRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeIntRangeFilter(String columnName, Condition condition, int value) { switch (condition) { case LESS_THAN: - return lt(columnName, RangeConditionFilter.parseIntFilter(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, RangeConditionFilter.parseIntFilter(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, RangeConditionFilter.parseIntFilter(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, RangeConditionFilter.parseIntFilter(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index d0718eab953..66c7916dc59 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -51,16 +51,16 @@ public LongRangeFilter(String columnName, long val1, long val2, boolean lowerInc } } - static WhereFilter makeLongRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeLongRangeFilter(String columnName, Condition condition, long value) { switch (condition) { case LESS_THAN: - return lt(columnName, RangeConditionFilter.parseLongFilter(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, RangeConditionFilter.parseLongFilter(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, RangeConditionFilter.parseLongFilter(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, RangeConditionFilter.parseLongFilter(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index c5a24c5bc5c..a88925fd583 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -33,6 +33,7 @@ import java.time.LocalTime; import java.time.ZonedDateTime; import java.util.*; +import java.util.function.Consumer; import java.util.stream.Stream; public class MatchFilter extends WhereFilterImpl implements DependencyStreamProvider { @@ -49,6 +50,8 @@ static MatchFilter ofLiterals( literals.stream().map(AsObject::of).toArray()); } + + @NotNull private final String columnName; private Object[] values; @@ -183,39 +186,16 @@ public synchronized void init( } final List valueList = new ArrayList<>(); final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); - final ColumnTypeConvertor convertor = - ColumnTypeConvertorFactory.getConvertor(column.getDataType(), column.getName()); + final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType()); for (String strValue : strValues) { - if (queryScopeVariables.containsKey(strValue)) { - Object paramValue = queryScopeVariables.get(strValue); - if (paramValue != null && paramValue.getClass().isArray()) { - ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); - for (int ai = 0; ai < accessor.length(); ++ai) { - valueList.add(convertor.convertParamValue(accessor.get(ai))); - } - } else if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { - for (final Object paramValueMember : (Collection) paramValue) { - valueList.add(convertor.convertParamValue(paramValueMember)); - } - } else { - valueList.add(convertor.convertParamValue(paramValue)); - } - } else { - Object convertedValue; - try { - convertedValue = convertor.convertStringLiteral(strValue); - } catch (Throwable t) { - throw new IllegalArgumentException("Failed to convert literal value <" + strValue + - "> for column \"" + columnName + "\" of type " + column.getDataType().getName(), t); - } - valueList.add(convertedValue); - } + convertor.convertValue(column, strValue, queryScopeVariables, valueList::add); } // values = (Object[])ArrayTypeUtils.toArray(valueList, TypeUtils.getBoxedType(theColumn.getDataType())); values = valueList.toArray(); initialized = true; } + @Override public SafeCloseable beginOperation(@NotNull final Table sourceTable) { if (initialDependenciesGathered || dataIndex != null) { @@ -286,10 +266,39 @@ Object convertParamValue(Object paramValue) { } return paramValue; } + + final void convertValue( + @NotNull final ColumnDefinition column, + @NotNull final String strValue, + @NotNull final Map queryScopeVariables, + @NotNull final Consumer valueConsumer) { + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); + if (paramValue != null && paramValue.getClass().isArray()) { + ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); + for (int ai = 0; ai < accessor.length(); ++ai) { + valueConsumer.accept(convertParamValue(accessor.get(ai))); + } + } else if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { + for (final Object paramValueMember : (Collection) paramValue) { + valueConsumer.accept(convertParamValue(paramValueMember)); + } + } else { + valueConsumer.accept(convertParamValue(paramValue)); + } + } else { + try { + valueConsumer.accept(convertStringLiteral(strValue)); + } catch (Throwable t) { + throw new IllegalArgumentException("Failed to convert literal value <" + strValue + + "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); + } + } + } } public static class ColumnTypeConvertorFactory { - public static ColumnTypeConvertor getConvertor(final Class cls, final String name) { + public static ColumnTypeConvertor getConvertor(final Class cls) { if (cls == byte.class) { return new ColumnTypeConvertor() { @Override @@ -469,9 +478,9 @@ Object convertStringLiteral(String str) { Object convertStringLiteral(String str) { if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( - "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + "LocalDate literal not enclosed in single-quotes (\"" + str + "\")"); } - return LocalDate.parse(str.substring(1, str.length() - 1)); + return DateTimeUtils.parseLocalDate(str.substring(1, str.length() - 1)); } }; } @@ -481,9 +490,9 @@ Object convertStringLiteral(String str) { Object convertStringLiteral(String str) { if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( - "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + "LocalTime literal not enclosed in single-quotes (\"" + str + "\")"); } - return LocalTime.parse(str.substring(1, str.length() - 1)); + return DateTimeUtils.parseLocalTime(str.substring(1, str.length() - 1)); } }; } @@ -493,7 +502,7 @@ Object convertStringLiteral(String str) { Object convertStringLiteral(String str) { if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( - "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + "LocalDateTime literal not enclosed in single-quotes (\"" + str + "\")"); } return LocalDateTime.parse(str.substring(1, str.length() - 1)); } @@ -505,9 +514,9 @@ Object convertStringLiteral(String str) { Object convertStringLiteral(String str) { if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( - "Instant literal not enclosed in single-quotes (\"" + str + "\")"); + "ZoneDateTime literal not enclosed in single-quotes (\"" + str + "\")"); } - return ZonedDateTime.parse(str.substring(1, str.length() - 1)); + return DateTimeUtils.parseZonedDateTime(str.substring(1, str.length() - 1)); } }; } @@ -553,7 +562,7 @@ Object convertStringLiteral(String str) { @Override Object convertStringLiteral(String str) { throw new IllegalArgumentException( - "Can't convert String to " + cls.getName() + " for MatchFilter value auto-conversion"); + "Can't create " + cls.getName() + " from String Literal for value auto-conversion"); } }; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index f7cdfaa570a..162b56a5538 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -13,10 +13,16 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.gui.table.filters.Condition; import io.deephaven.util.type.TypeUtils; +import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; import java.math.BigDecimal; import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZonedDateTime; import java.util.Collections; import java.util.List; @@ -59,7 +65,7 @@ public RangeConditionFilter(String columnName, Condition condition, String value * @param condition the condition for filtering * @param value a String representation of the numeric filter value * @param expression the original expression prior to being parsed - * @param parserConfiguration + * @param parserConfiguration the parser configuration to use */ public RangeConditionFilter(String columnName, Condition condition, String value, String expression, FormulaParserConfiguration parserConfiguration) { @@ -73,7 +79,7 @@ public RangeConditionFilter(String columnName, Condition condition, String value * @param conditionString the String representation of a condition for filtering * @param value a String representation of the numeric filter value * @param expression the original expression prior to being parsed - * @param parserConfiguration + * @param parserConfiguration the parser configuration to useyy */ public RangeConditionFilter(String columnName, String conditionString, String value, String expression, FormulaParserConfiguration parserConfiguration) { @@ -150,32 +156,52 @@ public void init( final Class colClass = def.getDataType(); + final MatchFilter.ColumnTypeConvertor convertor = + MatchFilter.ColumnTypeConvertorFactory.getConvertor(def.getDataType()); + + final MutableObject realValue = new MutableObject<>(); + convertor.convertValue(def, value, compilationProcessor.getQueryScopeVariables(), + parsedValue -> { + if (realValue.getValue() != null) { + throw new IllegalArgumentException(value + " is an array type"); + } + realValue.setValue(parsedValue); + }); + if (colClass == double.class || colClass == Double.class) { - filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, value); + filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, (double) realValue.getValue()); } else if (colClass == float.class || colClass == Float.class) { - filter = FloatRangeFilter.makeFloatRangeFilter(columnName, condition, value); + filter = FloatRangeFilter.makeFloatRangeFilter(columnName, condition, (float) realValue.getValue()); } else if (colClass == char.class || colClass == Character.class) { - filter = CharRangeFilter.makeCharRangeFilter(columnName, condition, value); + filter = CharRangeFilter.makeCharRangeFilter(columnName, condition, (char) realValue.getValue()); } else if (colClass == byte.class || colClass == Byte.class) { - filter = ByteRangeFilter.makeByteRangeFilter(columnName, condition, value); + filter = ByteRangeFilter.makeByteRangeFilter(columnName, condition, (byte) realValue.getValue()); } else if (colClass == short.class || colClass == Short.class) { - filter = ShortRangeFilter.makeShortRangeFilter(columnName, condition, value); + filter = ShortRangeFilter.makeShortRangeFilter(columnName, condition, (short) realValue.getValue()); } else if (colClass == int.class || colClass == Integer.class) { - filter = IntRangeFilter.makeIntRangeFilter(columnName, condition, value); + filter = IntRangeFilter.makeIntRangeFilter(columnName, condition, (int) realValue.getValue()); } else if (colClass == long.class || colClass == Long.class) { - filter = LongRangeFilter.makeLongRangeFilter(columnName, condition, value); - } else if (io.deephaven.util.type.TypeUtils.isDateTime(colClass)) { - filter = makeDateTimeRangeFilter(columnName, condition, value); + filter = LongRangeFilter.makeLongRangeFilter(columnName, condition, (long) realValue.getValue()); + } else if (colClass == Instant.class) { + filter = makeInstantRangeFilter(columnName, condition, + DateTimeUtils.epochNanos((Instant) realValue.getValue())); + } else if (colClass == LocalDate.class) { + filter = makeComparableRangeFilter(columnName, condition, (LocalDate) realValue.getValue()); + } else if (colClass == LocalTime.class) { + filter = makeComparableRangeFilter(columnName, condition, (LocalTime) realValue.getValue()); + } else if (colClass == LocalDateTime.class) { + filter = makeComparableRangeFilter(columnName, condition, (LocalDateTime) realValue.getValue()); + } else if (colClass == ZonedDateTime.class) { + filter = makeInstantRangeFilter(columnName, condition, + DateTimeUtils.epochNanos((ZonedDateTime) realValue.getValue())); } else if (BigDecimal.class.isAssignableFrom(colClass)) { - filter = makeComparableRangeFilter(columnName, condition, new BigDecimal(value)); + filter = makeComparableRangeFilter(columnName, condition, (BigDecimal) realValue.getValue()); } else if (BigInteger.class.isAssignableFrom(colClass)) { - filter = makeComparableRangeFilter(columnName, condition, new BigInteger(value)); + filter = makeComparableRangeFilter(columnName, condition, (BigInteger) realValue.getValue()); } else if (io.deephaven.util.type.TypeUtils.isString(colClass)) { - final String stringValue = MatchFilter.ColumnTypeConvertorFactory.getConvertor(String.class, columnName) - .convertStringLiteral(value).toString(); - filter = makeComparableRangeFilter(columnName, condition, stringValue); + filter = makeComparableRangeFilter(columnName, condition, (String) realValue.getValue()); } else if (TypeUtils.isBoxedBoolean(colClass) || colClass == boolean.class) { - filter = makeComparableRangeFilter(columnName, condition, Boolean.valueOf(value)); + filter = makeComparableRangeFilter(columnName, condition, (Boolean) realValue.getValue()); } else { // The expression looks like a comparison of number, string, or boolean // but the type does not match (or the column type is misconfigured) @@ -190,54 +216,21 @@ public void init( filter.init(tableDefinition, compilationProcessor); } - public static char parseCharFilter(String value) { - if (value.startsWith("'") && value.endsWith("'") && value.length() == 3) { - return value.charAt(1); - } - if (value.startsWith("\"") && value.endsWith("\"") && value.length() == 3) { - return value.charAt(1); - } - return (char) Long.parseLong(value); - } - - public static byte parseByteFilter(String value) { - return Byte.parseByte(value); - } - - public static short parseShortFilter(String value) { - return Short.parseShort(value); - } - - public static int parseIntFilter(String value) { - return Integer.parseInt(value); - } - - public static long parseLongFilter(String value) { - return Long.parseLong(value); - } - - private static LongRangeFilter makeDateTimeRangeFilter(String columnName, Condition condition, String value) { + private static LongRangeFilter makeInstantRangeFilter(String columnName, Condition condition, long value) { switch (condition) { case LESS_THAN: - return new InstantRangeFilter(columnName, parseInstantNanos(value), Long.MIN_VALUE, true, false); + return new InstantRangeFilter(columnName, value, Long.MIN_VALUE, true, false); case LESS_THAN_OR_EQUAL: - return new InstantRangeFilter(columnName, parseInstantNanos(value), Long.MIN_VALUE, true, true); + return new InstantRangeFilter(columnName, value, Long.MIN_VALUE, true, true); case GREATER_THAN: - return new InstantRangeFilter(columnName, parseInstantNanos(value), Long.MAX_VALUE, false, true); + return new InstantRangeFilter(columnName, value, Long.MAX_VALUE, false, true); case GREATER_THAN_OR_EQUAL: - return new InstantRangeFilter(columnName, parseInstantNanos(value), Long.MAX_VALUE, true, true); + return new InstantRangeFilter(columnName, value, Long.MAX_VALUE, true, true); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } } - private static long parseInstantNanos(String value) { - if (value.startsWith("'") && value.endsWith("'")) { - return DateTimeUtils.epochNanos(DateTimeUtils.parseInstant(value.substring(1, value.length() - 1))); - } - return Long.parseLong(value); - } - private static SingleSidedComparableRangeFilter makeComparableRangeFilter(String columnName, Condition condition, Comparable comparable) { switch (condition) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 461707e7b7f..801687aa593 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -51,16 +51,16 @@ public ShortRangeFilter(String columnName, short val1, short val2, boolean lower } } - static WhereFilter makeShortRangeFilter(String columnName, Condition condition, String value) { + static WhereFilter makeShortRangeFilter(String columnName, Condition condition, short value) { switch (condition) { case LESS_THAN: - return lt(columnName, RangeConditionFilter.parseShortFilter(value)); + return lt(columnName, value); case LESS_THAN_OR_EQUAL: - return leq(columnName, RangeConditionFilter.parseShortFilter(value)); + return leq(columnName, value); case GREATER_THAN: - return gt(columnName, RangeConditionFilter.parseShortFilter(value)); + return gt(columnName, value); case GREATER_THAN_OR_EQUAL: - return geq(columnName, RangeConditionFilter.parseShortFilter(value)); + return geq(columnName, value); default: throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index c021504f702..7cac7f26b49 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -7,6 +7,7 @@ import io.deephaven.api.filter.FilterPattern; import io.deephaven.api.filter.FilterPattern.Mode; import io.deephaven.base.Pair; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryScope; import io.deephaven.api.expression.AbstractExpressionFactory; import io.deephaven.engine.table.ColumnDefinition; @@ -74,15 +75,18 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a value); } }); - // == - // = - // != + // == + // = + // != + // < + // <= + // > parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + ID_PTRN + ")\\s*(?:(?:={1,2})|(!=))\\s*(" + ID_PTRN + ")" + END_PTRN) { + START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { @Override public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { final String columnName = matcher.group(1); - final boolean inverted = matcher.group(2) != null; + final String op = matcher.group(2); final String paramName = matcher.group(3); final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; @@ -92,18 +96,34 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a .append(expression).endl(); return ConditionFilter.createConditionFilter(expression, parserConfiguration); } - try { - QueryScope.getParamValue(paramName); - } catch (QueryScope.MissingVariableException e) { + if (!ExecutionContext.getContext().getQueryScope().hasParamName(paramName)) { return ConditionFilter.createConditionFilter(expression, parserConfiguration); } log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); - return new MatchFilter( - MatchFilter.CaseSensitivity.MatchCase, - inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, - columnName, - paramName); + + boolean inverted = false; + switch (op) { + case "!=": + inverted = true; + case "=": + case "==": + return new MatchFilter( + MatchFilter.CaseSensitivity.MatchCase, + inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, + columnName, + paramName); + case "<": + case ">": + case "<=": + case ">=": + log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") + .append(expression).endl(); + return new RangeConditionFilter(columnName, op, paramName, expression, + parserConfiguration); + default: + throw new IllegalStateException("Unexpected operator: " + op); + } } }); @@ -396,7 +416,7 @@ public static WhereFilter stringContainsFilter( boolean removeQuotes, String... values) { final String value = - constructStringContainsRegex(values, matchType, internalDisjunctive, removeQuotes, columnName); + constructStringContainsRegex(values, matchType, internalDisjunctive, removeQuotes); return WhereFilterAdapter.of(FilterPattern.of( ColumnName.of(columnName), Pattern.compile(value, sensitivity == CaseSensitivity.IgnoreCase ? Pattern.CASE_INSENSITIVE : 0), @@ -408,14 +428,13 @@ private static String constructStringContainsRegex( String[] values, MatchType matchType, boolean internalDisjunctive, - boolean removeQuotes, - String columnName) { + boolean removeQuotes) { if (values == null || values.length == 0) { throw new IllegalArgumentException( "constructStringContainsRegex must be called with at least one value parameter"); } final MatchFilter.ColumnTypeConvertor converter = removeQuotes - ? MatchFilter.ColumnTypeConvertorFactory.getConvertor(String.class, columnName) + ? MatchFilter.ColumnTypeConvertorFactory.getConvertor(String.class) : null; final String regex; final Stream valueStream = Arrays.stream(values) From 480bedfee847ebf35d24e7afd77ef33fb27b5aa1 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 13:59:56 -0600 Subject: [PATCH 04/44] Fix ZonedDateTime and Array Shenanigans --- .../engine/table/impl/select/MatchFilter.java | 13 ++- .../impl/select/RangeConditionFilter.java | 28 +++-- .../impl/select/ZonedDateTimeRangeFilter.java | 109 ++++++++++++++++++ 3 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index a88925fd583..a4b8a25e71d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -267,7 +267,15 @@ Object convertParamValue(Object paramValue) { return paramValue; } - final void convertValue( + /** + * Convert the string value to the appropriate type for the column. + * @param column the column definition + * @param strValue the string value to convert + * @param queryScopeVariables the query scope variables + * @param valueConsumer the consumer for the converted value + * @return whether the value was an array or collection + */ + final boolean convertValue( @NotNull final ColumnDefinition column, @NotNull final String strValue, @NotNull final Map queryScopeVariables, @@ -279,10 +287,12 @@ final void convertValue( for (int ai = 0; ai < accessor.length(); ++ai) { valueConsumer.accept(convertParamValue(accessor.get(ai))); } + return true; } else if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { for (final Object paramValueMember : (Collection) paramValue) { valueConsumer.accept(convertParamValue(paramValueMember)); } + return true; } else { valueConsumer.accept(convertParamValue(paramValue)); } @@ -294,6 +304,7 @@ final void convertValue( "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); } } + return false; } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index 162b56a5538..ae357b93981 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -160,13 +160,12 @@ public void init( MatchFilter.ColumnTypeConvertorFactory.getConvertor(def.getDataType()); final MutableObject realValue = new MutableObject<>(); - convertor.convertValue(def, value, compilationProcessor.getQueryScopeVariables(), - parsedValue -> { - if (realValue.getValue() != null) { - throw new IllegalArgumentException(value + " is an array type"); - } - realValue.setValue(parsedValue); - }); + boolean wasAnArrayType = convertor.convertValue( + def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); + if (wasAnArrayType) { + throw new IllegalArgumentException("RangeConditionFilter does not support array types for column " + + columnName + " with value <" + value + ">"); + } if (colClass == double.class || colClass == Double.class) { filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, (double) realValue.getValue()); @@ -231,6 +230,21 @@ private static LongRangeFilter makeInstantRangeFilter(String columnName, Conditi } } + private static LongRangeFilter makeZonedDateTimeRangeFilter(String columnName, Condition condition, long value) { + switch (condition) { + case LESS_THAN: + return new ZonedDateTimeRangeFilter(columnName, value, Long.MIN_VALUE, true, false); + case LESS_THAN_OR_EQUAL: + return new ZonedDateTimeRangeFilter(columnName, value, Long.MIN_VALUE, true, true); + case GREATER_THAN: + return new ZonedDateTimeRangeFilter(columnName, value, Long.MAX_VALUE, false, true); + case GREATER_THAN_OR_EQUAL: + return new ZonedDateTimeRangeFilter(columnName, value, Long.MAX_VALUE, true, true); + default: + throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + } + } + private static SingleSidedComparableRangeFilter makeComparableRangeFilter(String columnName, Condition condition, Comparable comparable) { switch (condition) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java new file mode 100644 index 00000000000..bf3e292c289 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java @@ -0,0 +1,109 @@ +// +// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.table.impl.select; + +import io.deephaven.base.verify.Assert; +import io.deephaven.engine.table.ColumnDefinition; +import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; +import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; +import io.deephaven.engine.table.ColumnSource; +import io.deephaven.engine.table.impl.sources.ReinterpretUtils; +import io.deephaven.chunk.*; +import io.deephaven.chunk.attributes.Values; +import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.rowset.RowSet; +import io.deephaven.time.DateTimeUtils; +import org.jetbrains.annotations.NotNull; + +import java.time.ZonedDateTime; + +public class ZonedDateTimeRangeFilter extends LongRangeFilter { + + public ZonedDateTimeRangeFilter(String columnName, ZonedDateTime val1, ZonedDateTime val2) { + super(columnName, DateTimeUtils.epochNanos(val1), DateTimeUtils.epochNanos(val2), true, true); + } + + public ZonedDateTimeRangeFilter( + String columnName, ZonedDateTime val1, ZonedDateTime val2, boolean lowerInclusive, boolean upperInclusive) { + super(columnName, DateTimeUtils.epochNanos(val1), DateTimeUtils.epochNanos(val2), + lowerInclusive, upperInclusive); + } + + public ZonedDateTimeRangeFilter( + String columnName, long val1, long val2, boolean lowerInclusive, boolean upperInclusive) { + super(columnName, val1, val2, lowerInclusive, upperInclusive); + } + + @Override + public void init(@NotNull final TableDefinition tableDefinition) { + if (chunkFilter != null) { + return; + } + + final ColumnDefinition def = tableDefinition.getColumn(columnName); + if (def == null) { + throw new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " + + tableDefinition.getColumnNames()); + } + + final Class colClass = def.getDataType(); + Assert.eq(colClass, "colClass", ZonedDateTime.class); + + longFilter = super.initChunkFilter(); + chunkFilter = new ZonedDateTimeLongChunkFilterAdapter(); + } + + @Override + public ZonedDateTimeRangeFilter copy() { + final ZonedDateTimeRangeFilter copy = + new ZonedDateTimeRangeFilter(columnName, lower, upper, lowerInclusive, upperInclusive); + copy.chunkFilter = chunkFilter; + copy.longFilter = longFilter; + return copy; + } + + @Override + public String toString() { + return "ZonedDateTimeRangeFilter(" + columnName + " in " + + (lowerInclusive ? "[" : "(") + + DateTimeUtils.epochNanosToInstant(lower) + "," + DateTimeUtils.epochNanosToInstant(upper) + + (upperInclusive ? "]" : ")") + ")"; + } + + @NotNull + @Override + WritableRowSet binarySearch( + @NotNull final RowSet selection, + @NotNull final ColumnSource columnSource, + final boolean usePrev, + final boolean reverse) { + if (selection.isEmpty()) { + return selection.copy(); + } + + // noinspection unchecked + final ColumnSource instantColumnSource = + ReinterpretUtils.zonedDateTimeToLongSource((ColumnSource) columnSource); + return super.binarySearch(selection, instantColumnSource, usePrev, reverse); + } + + private class ZonedDateTimeLongChunkFilterAdapter implements ChunkFilter { + @Override + public void filter(Chunk values, LongChunk keys, + WritableLongChunk results) { + try (final WritableLongChunk writableLongChunk = + WritableLongChunk.makeWritableChunk(values.size())) { + + final ObjectChunk objectValues = values.asObjectChunk(); + for (int ii = 0; ii < values.size(); ++ii) { + final ZonedDateTime instant = objectValues.get(ii); + writableLongChunk.set(ii, DateTimeUtils.epochNanos(instant)); + } + writableLongChunk.setSize(values.size()); + longFilter.filter(writableLongChunk, keys, results); + } + } + } +} From 45e69bb570f3d66b1e1d0fa5563b5b8666dd7df6 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 14:05:01 -0600 Subject: [PATCH 05/44] cleanup personal review --- .../deephaven/engine/table/impl/select/MatchFilter.java | 3 --- .../engine/table/impl/select/WhereFilterFactory.java | 4 ++-- .../table/impl/select/ZonedDateTimeRangeFilter.java | 8 ++++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index a4b8a25e71d..de91b4011bc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -50,8 +50,6 @@ static MatchFilter ofLiterals( literals.stream().map(AsObject::of).toArray()); } - - @NotNull private final String columnName; private Object[] values; @@ -195,7 +193,6 @@ public synchronized void init( initialized = true; } - @Override public SafeCloseable beginOperation(@NotNull final Table sourceTable) { if (initialDependenciesGathered || dataIndex != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 7cac7f26b49..fe328052726 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -99,8 +99,6 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a if (!ExecutionContext.getContext().getQueryScope().hasParamName(paramName)) { return ConditionFilter.createConditionFilter(expression, parserConfiguration); } - log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) - .endl(); boolean inverted = false; switch (op) { @@ -108,6 +106,8 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a inverted = true; case "=": case "==": + log.debug().append("WhereFilterFactory creating MatchFilter for expression: ") + .append(expression).endl(); return new MatchFilter( MatchFilter.CaseSensitivity.MatchCase, inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java index bf3e292c289..15d310d0b0e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java @@ -84,9 +84,9 @@ WritableRowSet binarySearch( } // noinspection unchecked - final ColumnSource instantColumnSource = + final ColumnSource zdtColumnSource = ReinterpretUtils.zonedDateTimeToLongSource((ColumnSource) columnSource); - return super.binarySearch(selection, instantColumnSource, usePrev, reverse); + return super.binarySearch(selection, zdtColumnSource, usePrev, reverse); } private class ZonedDateTimeLongChunkFilterAdapter implements ChunkFilter { @@ -98,8 +98,8 @@ public void filter(Chunk values, LongChunk key final ObjectChunk objectValues = values.asObjectChunk(); for (int ii = 0; ii < values.size(); ++ii) { - final ZonedDateTime instant = objectValues.get(ii); - writableLongChunk.set(ii, DateTimeUtils.epochNanos(instant)); + final ZonedDateTime zdt = objectValues.get(ii); + writableLongChunk.set(ii, DateTimeUtils.epochNanos(zdt)); } writableLongChunk.setSize(values.size()); longFilter.filter(writableLongChunk, keys, results); From bd535813b10eab4a7cea0f4b457684adaec255be Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 14:06:20 -0600 Subject: [PATCH 06/44] spotless --- .../io/deephaven/engine/table/impl/select/MatchFilter.java | 1 + .../engine/table/impl/select/WhereFilterFactory.java | 6 +++--- .../engine/table/impl/select/ZonedDateTimeRangeFilter.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index de91b4011bc..c1530e31892 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -266,6 +266,7 @@ Object convertParamValue(Object paramValue) { /** * Convert the string value to the appropriate type for the column. + * * @param column the column definition * @param strValue the string value to convert * @param queryScopeVariables the query scope variables diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index fe328052726..750d22027a7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -76,11 +76,11 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a } }); // == - // = + // = // != - // < + // < // <= - // > + // > parser.registerFactory(new AbstractExpressionFactory<>( START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java index 15d310d0b0e..ac44e7b8504 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java @@ -92,9 +92,9 @@ WritableRowSet binarySearch( private class ZonedDateTimeLongChunkFilterAdapter implements ChunkFilter { @Override public void filter(Chunk values, LongChunk keys, - WritableLongChunk results) { + WritableLongChunk results) { try (final WritableLongChunk writableLongChunk = - WritableLongChunk.makeWritableChunk(values.size())) { + WritableLongChunk.makeWritableChunk(values.size())) { final ObjectChunk objectValues = values.asObjectChunk(); for (int ii = 0; ii < values.size(); ++ii) { From 0786673a8b5930180e2d8c415d6cea5c0022d53c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 7 Jun 2024 17:15:05 -0600 Subject: [PATCH 07/44] Allow Inverted Versions of Filter Regex --- .../impl/select/RangeConditionFilter.java | 25 ++- .../table/impl/select/WhereFilterFactory.java | 177 ++++++++++-------- .../engine/table/impl/QueryFactory.java | 2 +- .../table/impl/QueryTableWhereTest.java | 42 +++-- 4 files changed, 143 insertions(+), 103 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index ae357b93981..c9a53ac1413 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -159,15 +159,26 @@ public void init( final MatchFilter.ColumnTypeConvertor convertor = MatchFilter.ColumnTypeConvertorFactory.getConvertor(def.getDataType()); + RuntimeException conversionError = null; final MutableObject realValue = new MutableObject<>(); - boolean wasAnArrayType = convertor.convertValue( - def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); - if (wasAnArrayType) { - throw new IllegalArgumentException("RangeConditionFilter does not support array types for column " - + columnName + " with value <" + value + ">"); + try { + boolean wasAnArrayType = convertor.convertValue( + def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); + if (wasAnArrayType) { + throw new IllegalArgumentException("RangeConditionFilter does not support array types for column " + + columnName + " with value <" + value + ">"); + } + } catch (final RuntimeException err) { + conversionError = err; } - if (colClass == double.class || colClass == Double.class) { + if (conversionError != null) { + if (expression != null) { + filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + } else { + throw conversionError; + } + } else if (colClass == double.class || colClass == Double.class) { filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, (double) realValue.getValue()); } else if (colClass == float.class || colClass == Float.class) { filter = FloatRangeFilter.makeFloatRangeFilter(columnName, condition, (float) realValue.getValue()); @@ -191,7 +202,7 @@ public void init( } else if (colClass == LocalDateTime.class) { filter = makeComparableRangeFilter(columnName, condition, (LocalDateTime) realValue.getValue()); } else if (colClass == ZonedDateTime.class) { - filter = makeInstantRangeFilter(columnName, condition, + filter = makeZonedDateTimeRangeFilter(columnName, condition, DateTimeUtils.epochNanos((ZonedDateTime) realValue.getValue())); } else if (BigDecimal.class.isAssignableFrom(colClass)) { filter = makeComparableRangeFilter(columnName, condition, (BigDecimal) realValue.getValue()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 750d22027a7..0e792705dbe 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -49,38 +49,55 @@ public class WhereFilterFactory { private static final ExpressionParser parser = new ExpressionParser<>(); static { - // == - // = - // != + // == + // = + // != + // < + // <= + // > + // >= parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + ID_PTRN + ")\\s*(?:(?:={1,2})|(!=))\\s*(" + LITERAL_PTRN + ")" + END_PTRN) { + START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + LITERAL_PTRN + ")" + END_PTRN) { @Override public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { final String columnName = matcher.group(1); - final boolean inverted = matcher.group(2) != null; + final String op = matcher.group(2); final String value = matcher.group(3); + final boolean mirrored = false; - final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; - if (isRowVariable(columnName)) { - log.debug().append("WhereFilterFactory creating ConditionFilter for expression: ") - .append(expression).endl(); - return ConditionFilter.createConditionFilter(expression, parserConfiguration); - } - log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) - .endl(); - return new MatchFilter( - MatchFilter.CaseSensitivity.MatchCase, - inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, - columnName, - value); + return getWhereFilterOneSideColumn( + expression, (FormulaParserConfiguration) args[0], columnName, op, value, mirrored); + } + }); + + // == + // = + // != + // < + // <= + // > + // >= + parser.registerFactory(new AbstractExpressionFactory<>( + START_PTRN + "(" + LITERAL_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { + @Override + public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { + final String value = matcher.group(1); + final String op = matcher.group(6); + final String columnName = matcher.group(7); + final boolean mirrored = true; + + return getWhereFilterOneSideColumn( + expression, (FormulaParserConfiguration) args[0], columnName, op, value, mirrored); } }); + // == // = // != // < // <= // > + // >= parser.registerFactory(new AbstractExpressionFactory<>( START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { @Override @@ -88,73 +105,20 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a final String columnName = matcher.group(1); final String op = matcher.group(2); final String paramName = matcher.group(3); - final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; - if (isRowVariable(columnName)) { - log.debug().append("WhereFilterFactory creating ConditionFilter for expression: ") - .append(expression).endl(); - return ConditionFilter.createConditionFilter(expression, parserConfiguration); - } - if (!ExecutionContext.getContext().getQueryScope().hasParamName(paramName)) { - return ConditionFilter.createConditionFilter(expression, parserConfiguration); + boolean mirrored = false; + final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + if (!queryScope.hasParamName(paramName)) { + if (queryScope.hasParamName(columnName)) { + mirrored = true; + } else { + return ConditionFilter.createConditionFilter(expression, parserConfiguration); + } } - boolean inverted = false; - switch (op) { - case "!=": - inverted = true; - case "=": - case "==": - log.debug().append("WhereFilterFactory creating MatchFilter for expression: ") - .append(expression).endl(); - return new MatchFilter( - MatchFilter.CaseSensitivity.MatchCase, - inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, - columnName, - paramName); - case "<": - case ">": - case "<=": - case ">=": - log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") - .append(expression).endl(); - return new RangeConditionFilter(columnName, op, paramName, expression, - parserConfiguration); - default: - throw new IllegalStateException("Unexpected operator: " + op); - } - } - }); - - // < - // <= - // > - // >= - parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + ID_PTRN + ")\\s*([<>]=?)\\s*(" + LITERAL_PTRN + ")" + END_PTRN) { - @Override - public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { - final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; - final String columnName = matcher.group(1); - final String conditionString = matcher.group(2); - final String value = matcher.group(3); - if (isRowVariable(columnName)) { - log.debug().append("WhereFilterFactory creating ConditionFilter for expression: ") - .append(expression).endl(); - return ConditionFilter.createConditionFilter(expression, parserConfiguration); - } - try { - log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") - .append(expression).endl(); - return new RangeConditionFilter(columnName, conditionString, value, expression, - parserConfiguration); - } catch (Exception e) { - log.warn().append("WhereFilterFactory could not make RangeFilter for expression: ") - .append(expression).append(" due to ").append(e) - .append(" Creating ConditionFilter instead.").endl(); - return ConditionFilter.createConditionFilter(expression, parserConfiguration); - } + return getWhereFilterOneSideColumn( + expression, parserConfiguration, columnName, op, paramName, mirrored); } }); @@ -222,6 +186,57 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a }); } + private static @NotNull WhereFilter getWhereFilterOneSideColumn( + final String expression, + final FormulaParserConfiguration parserConfiguration, + final String columnName, + String op, + final String value, + boolean mirrored) { + + if (isRowVariable(columnName)) { + log.debug().append("WhereFilterFactory creating ConditionFilter for expression: ") + .append(expression).endl(); + return ConditionFilter.createConditionFilter(expression, parserConfiguration); + } + + switch (op) { + case "!=": + mirrored = !mirrored; + case "=": + case "==": + log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) + .endl(); + return new MatchFilter( + CaseSensitivity.MatchCase, + mirrored ? MatchType.Inverted : MatchType.Regular, + columnName, + value); + + case "<": + case ">": + case "<=": + case ">=": + try { + if (mirrored) { + final String dir = op.substring(0, 1); + op = op.replaceFirst(dir, dir.equals("<") ? ">" : "<"); + } + log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") + .append(expression).endl(); + return new RangeConditionFilter(columnName, op, value, expression, parserConfiguration); + } catch (Exception e) { + log.warn().append("WhereFilterFactory could not make RangeFilter for expression: ") + .append(expression).append(" due to ").append(e) + .append(" Creating ConditionFilter instead.").endl(); + return ConditionFilter.createConditionFilter(expression, parserConfiguration); + } + + default: + throw new IllegalStateException("Unexpected operator: " + op); + } + } + private static boolean isRowVariable(String columnName) { return columnName.equals("i") || columnName.equals("ii") || columnName.equals("k"); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryFactory.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryFactory.java index 62c25a31b03..1e40d00084e 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryFactory.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryFactory.java @@ -664,7 +664,7 @@ private String createWhereFilter(Random random) { switch (columnTypes[colNum].getSimpleName()) { case "Instant": - filter.append(colName).append(" > ").append(random.nextInt(1000) * 1_000_000_000L); + filter.append(colName).append(" > '").append(random.nextInt(1000) * 1_000_000_000L).append("'"); break; case "String": diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index dafc65cfe97..0a8b54965b7 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -46,6 +46,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.time.Instant; +import java.time.ZonedDateTime; import java.util.Collections; import java.util.Random; import java.util.concurrent.CountDownLatch; @@ -1152,6 +1153,19 @@ public void testComparableBinarySearch() { QueryScope.addParam("nine", null); } + @Test + public void testZonedDateRangeFilter() { + final ZonedDateTime startTime = DateTimeUtils.parseZonedDateTime("2021-04-23T09:30 NY"); + final ZonedDateTime[] array = new ZonedDateTime[10]; + for (int ii = 0; ii < array.length; ++ii) { + array[ii] = DateTimeUtils.plus(startTime, 60_000_000_000L * ii); + } + final Table table = TableTools.newTable(col("ZDT", array)); + showWithRowSet(table); + + testRangeFilterHelper(table, "ZDT", array[5]); + } + @Test public void testInstantRangeFilter() { final Instant startTime = DateTimeUtils.parseInstant("2021-04-23T09:30 NY"); @@ -1162,11 +1176,7 @@ public void testInstantRangeFilter() { final Table table = TableTools.newTable(col("DT", array)); showWithRowSet(table); - final Table sorted = table.sort("DT"); - final Table backwards = table.sort("DT"); - - assertTableEquals(sorted.where("DT < '" + array[5] + "'"), sorted.where("ii < 5")); - assertTableEquals(backwards.where("DT < '" + array[5] + "'"), backwards.where("ii < 5")); + testRangeFilterHelper(table, "DT", array[5]); } @Test @@ -1184,22 +1194,26 @@ public void testCharRangeFilter() { final Table table = TableTools.newTable(charCol("CH", array)); showWithRowSet(table); - final Table sorted = table.sort("CH"); - final Table backwards = table.sort("CH"); + testRangeFilterHelper(table, "CH", array[5]); + } + + private void testRangeFilterHelper(Table table, String name, T mid) { + final Table sorted = table.sort(name); + final Table backwards = table.sort(name); showWithRowSet(sorted); - log.debug().append("Pivot: " + array[5]).endl(); + log.debug().append("Pivot: " + mid).endl(); - final Table rangeFiltered = sorted.where("CH < '" + array[5] + "'"); - final Table standardFiltered = sorted.where("'" + array[5] + "' > CH"); + final Table rangeFiltered = sorted.where(name + " < '" + mid + "'"); + final Table standardFiltered = sorted.where("'" + mid + "' > " + name); showWithRowSet(rangeFiltered); showWithRowSet(standardFiltered); assertTableEquals(rangeFiltered, standardFiltered); - assertTableEquals(backwards.where("CH < '" + array[5] + "'"), backwards.where("'" + array[5] + "' > CH")); - assertTableEquals(backwards.where("CH <= '" + array[5] + "'"), backwards.where("'" + array[5] + "' >= CH")); - assertTableEquals(backwards.where("CH > '" + array[5] + "'"), backwards.where("'" + array[5] + "' < CH")); - assertTableEquals(backwards.where("CH >= '" + array[5] + "'"), backwards.where("'" + array[5] + "' <= CH")); + assertTableEquals(backwards.where(name + " < '" + mid + "'"), backwards.where("'" + mid + "' > " + name)); + assertTableEquals(backwards.where(name + " <= '" + mid + "'"), backwards.where("'" + mid + "' >= " + name)); + assertTableEquals(backwards.where(name + " > '" + mid + "'"), backwards.where("'" + mid + "' < " + name)); + assertTableEquals(backwards.where(name + " >= '" + mid + "'"), backwards.where("'" + mid + "' <= " + name)); } @Test From b21007639c4d25b65c7583766e755b927325f291 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 11 Jun 2024 10:41:01 -0600 Subject: [PATCH 08/44] Rename RangeFilter --- ...eConditionFilter.java => RangeFilter.java} | 18 +++--- .../table/impl/select/WhereFilterAdapter.java | 20 +++--- .../table/impl/select/WhereFilterFactory.java | 2 +- .../table/impl/select/WhereFilterTest.java | 64 +++++++++---------- .../table/ops/filter/FilterFactory.java | 4 +- 5 files changed, 54 insertions(+), 54 deletions(-) rename engine/table/src/main/java/io/deephaven/engine/table/impl/select/{RangeConditionFilter.java => RangeFilter.java} (94%) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java similarity index 94% rename from engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java rename to engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index c9a53ac1413..74cf6cb9f3b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -35,7 +35,7 @@ *
  • GREATER_THAN_OR_EQUAL
    • * */ -public class RangeConditionFilter extends WhereFilterImpl { +public class RangeFilter extends WhereFilterImpl { private final String columnName; private final Condition condition; @@ -54,7 +54,7 @@ public class RangeConditionFilter extends WhereFilterImpl { * @param condition the condition for filtering * @param value a String representation of the numeric filter value */ - public RangeConditionFilter(String columnName, Condition condition, String value) { + public RangeFilter(String columnName, Condition condition, String value) { this(columnName, condition, value, null, null, null); } @@ -67,8 +67,8 @@ public RangeConditionFilter(String columnName, Condition condition, String value * @param expression the original expression prior to being parsed * @param parserConfiguration the parser configuration to use */ - public RangeConditionFilter(String columnName, Condition condition, String value, String expression, - FormulaParserConfiguration parserConfiguration) { + public RangeFilter(String columnName, Condition condition, String value, String expression, + FormulaParserConfiguration parserConfiguration) { this(columnName, condition, value, expression, null, parserConfiguration); } @@ -81,14 +81,14 @@ public RangeConditionFilter(String columnName, Condition condition, String value * @param expression the original expression prior to being parsed * @param parserConfiguration the parser configuration to useyy */ - public RangeConditionFilter(String columnName, String conditionString, String value, String expression, - FormulaParserConfiguration parserConfiguration) { + public RangeFilter(String columnName, String conditionString, String value, String expression, + FormulaParserConfiguration parserConfiguration) { this(columnName, conditionFromString(conditionString), value, expression, parserConfiguration); } // Used for copy method - private RangeConditionFilter(String columnName, Condition condition, String value, String expression, - WhereFilter filter, FormulaParserConfiguration parserConfiguration) { + private RangeFilter(String columnName, Condition condition, String value, String expression, + WhereFilter filter, FormulaParserConfiguration parserConfiguration) { Assert.eqTrue(conditionSupported(condition), condition + " is not supported by RangeConditionFilter"); this.columnName = columnName; this.condition = condition; @@ -296,7 +296,7 @@ public void setRecomputeListener(RecomputeListener listener) {} @Override public WhereFilter copy() { - return new RangeConditionFilter(columnName, condition, value, expression, filter, parserConfiguration); + return new RangeFilter(columnName, condition, value, expression, filter, parserConfiguration); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java index 70c6c040974..cab22f8e792 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java @@ -368,34 +368,34 @@ public WhereFilter visit(RawString rawString) { return original(); } - private RangeConditionFilter range(Object rhsLiteral) { + private RangeFilter range(Object rhsLiteral) { // TODO(deephaven-core#3730): More efficient io.deephaven.api.filter.FilterComparison to RangeFilter final String rhsLiteralAsStr = rhsLiteral.toString(); switch (preferred.operator()) { case LESS_THAN: - return new RangeConditionFilter(lhs.name(), Condition.LESS_THAN, rhsLiteralAsStr); + return new RangeFilter(lhs.name(), Condition.LESS_THAN, rhsLiteralAsStr); case LESS_THAN_OR_EQUAL: - return new RangeConditionFilter(lhs.name(), Condition.LESS_THAN_OR_EQUAL, rhsLiteralAsStr); + return new RangeFilter(lhs.name(), Condition.LESS_THAN_OR_EQUAL, rhsLiteralAsStr); case GREATER_THAN: - return new RangeConditionFilter(lhs.name(), Condition.GREATER_THAN, rhsLiteralAsStr); + return new RangeFilter(lhs.name(), Condition.GREATER_THAN, rhsLiteralAsStr); case GREATER_THAN_OR_EQUAL: - return new RangeConditionFilter(lhs.name(), Condition.GREATER_THAN_OR_EQUAL, rhsLiteralAsStr); + return new RangeFilter(lhs.name(), Condition.GREATER_THAN_OR_EQUAL, rhsLiteralAsStr); } throw new IllegalStateException("Unexpected"); } - private RangeConditionFilter range(String rhsLiteral) { + private RangeFilter range(String rhsLiteral) { // TODO(deephaven-core#3730): More efficient io.deephaven.api.filter.FilterComparison to RangeFilter final String quotedRhsLiteral = '"' + rhsLiteral + '"'; switch (preferred.operator()) { case LESS_THAN: - return new RangeConditionFilter(lhs.name(), Condition.LESS_THAN, quotedRhsLiteral); + return new RangeFilter(lhs.name(), Condition.LESS_THAN, quotedRhsLiteral); case LESS_THAN_OR_EQUAL: - return new RangeConditionFilter(lhs.name(), Condition.LESS_THAN_OR_EQUAL, quotedRhsLiteral); + return new RangeFilter(lhs.name(), Condition.LESS_THAN_OR_EQUAL, quotedRhsLiteral); case GREATER_THAN: - return new RangeConditionFilter(lhs.name(), Condition.GREATER_THAN, quotedRhsLiteral); + return new RangeFilter(lhs.name(), Condition.GREATER_THAN, quotedRhsLiteral); case GREATER_THAN_OR_EQUAL: - return new RangeConditionFilter(lhs.name(), Condition.GREATER_THAN_OR_EQUAL, quotedRhsLiteral); + return new RangeFilter(lhs.name(), Condition.GREATER_THAN_OR_EQUAL, quotedRhsLiteral); } throw new IllegalStateException("Unexpected"); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 0e792705dbe..ce4d0b79ae8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -224,7 +224,7 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a } log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") .append(expression).endl(); - return new RangeConditionFilter(columnName, op, value, expression, parserConfiguration); + return new RangeFilter(columnName, op, value, expression, parserConfiguration); } catch (Exception e) { log.warn().append("WhereFilterFactory could not make RangeFilter for expression: ") .append(expression).append(" due to ").append(e) diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java index 4ce85050b27..a1aaec275da 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java @@ -97,89 +97,89 @@ public void testNeq() { } public void testGt() { - regular(FilterComparison.gt(FOO, V42), RangeConditionFilter.class, + regular(FilterComparison.gt(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo greater than 42)"); - regular(FilterComparison.gt(V42, FOO), RangeConditionFilter.class, + regular(FilterComparison.gt(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than 42)"); - regular(FilterComparison.gt(FOO, HELLO), RangeConditionFilter.class, + regular(FilterComparison.gt(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo greater than \"Hello\")"); - regular(FilterComparison.gt(HELLO, FOO), RangeConditionFilter.class, + regular(FilterComparison.gt(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than \"Hello\")"); regular(FilterComparison.gt(FOO, BAR), ConditionFilter.class, "Foo > Bar"); - inverse(FilterComparison.gt(FOO, V42), RangeConditionFilter.class, + inverse(FilterComparison.gt(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to 42)"); - inverse(FilterComparison.gt(V42, FOO), RangeConditionFilter.class, + inverse(FilterComparison.gt(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to 42)"); - inverse(FilterComparison.gt(FOO, HELLO), RangeConditionFilter.class, + inverse(FilterComparison.gt(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to \"Hello\")"); - inverse(FilterComparison.gt(HELLO, FOO), RangeConditionFilter.class, + inverse(FilterComparison.gt(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); inverse(FilterComparison.gt(FOO, BAR), ConditionFilter.class, "Foo <= Bar"); } public void testGte() { - regular(FilterComparison.geq(FOO, V42), RangeConditionFilter.class, + regular(FilterComparison.geq(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to 42)"); - regular(FilterComparison.geq(V42, FOO), RangeConditionFilter.class, + regular(FilterComparison.geq(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to 42)"); - regular(FilterComparison.geq(FOO, HELLO), RangeConditionFilter.class, + regular(FilterComparison.geq(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); - regular(FilterComparison.geq(HELLO, FOO), RangeConditionFilter.class, + regular(FilterComparison.geq(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to \"Hello\")"); regular(FilterComparison.geq(FOO, BAR), ConditionFilter.class, "Foo >= Bar"); - inverse(FilterComparison.geq(FOO, V42), RangeConditionFilter.class, + inverse(FilterComparison.geq(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo less than 42)"); - inverse(FilterComparison.geq(V42, FOO), RangeConditionFilter.class, + inverse(FilterComparison.geq(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than 42)"); - inverse(FilterComparison.geq(FOO, HELLO), RangeConditionFilter.class, + inverse(FilterComparison.geq(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo less than \"Hello\")"); - inverse(FilterComparison.geq(HELLO, FOO), RangeConditionFilter.class, + inverse(FilterComparison.geq(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than \"Hello\")"); inverse(FilterComparison.geq(FOO, BAR), ConditionFilter.class, "Foo < Bar"); } public void testLt() { - regular(FilterComparison.lt(FOO, V42), RangeConditionFilter.class, + regular(FilterComparison.lt(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo less than 42)"); - regular(FilterComparison.lt(V42, FOO), RangeConditionFilter.class, + regular(FilterComparison.lt(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than 42)"); - regular(FilterComparison.lt(FOO, HELLO), RangeConditionFilter.class, + regular(FilterComparison.lt(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo less than \"Hello\")"); - regular(FilterComparison.lt(HELLO, FOO), RangeConditionFilter.class, + regular(FilterComparison.lt(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than \"Hello\")"); regular(FilterComparison.lt(FOO, BAR), ConditionFilter.class, "Foo < Bar"); - inverse(FilterComparison.lt(FOO, V42), RangeConditionFilter.class, + inverse(FilterComparison.lt(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to 42)"); - inverse(FilterComparison.lt(V42, FOO), RangeConditionFilter.class, + inverse(FilterComparison.lt(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to 42)"); - inverse(FilterComparison.lt(FOO, HELLO), RangeConditionFilter.class, + inverse(FilterComparison.lt(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); - inverse(FilterComparison.lt(HELLO, FOO), RangeConditionFilter.class, + inverse(FilterComparison.lt(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to \"Hello\")"); inverse(FilterComparison.lt(FOO, BAR), ConditionFilter.class, "Foo >= Bar"); } public void testLte() { - regular(FilterComparison.leq(FOO, V42), RangeConditionFilter.class, + regular(FilterComparison.leq(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to 42)"); - regular(FilterComparison.leq(V42, FOO), RangeConditionFilter.class, + regular(FilterComparison.leq(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to 42)"); - regular(FilterComparison.leq(FOO, HELLO), RangeConditionFilter.class, + regular(FilterComparison.leq(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo less than or equal to \"Hello\")"); - regular(FilterComparison.leq(HELLO, FOO), RangeConditionFilter.class, + regular(FilterComparison.leq(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); regular(FilterComparison.leq(FOO, BAR), ConditionFilter.class, "Foo <= Bar"); - inverse(FilterComparison.leq(FOO, V42), RangeConditionFilter.class, + inverse(FilterComparison.leq(FOO, V42), RangeFilter.class, "RangeConditionFilter(Foo greater than 42)"); - inverse(FilterComparison.leq(V42, FOO), RangeConditionFilter.class, + inverse(FilterComparison.leq(V42, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than 42)"); - inverse(FilterComparison.leq(FOO, HELLO), RangeConditionFilter.class, + inverse(FilterComparison.leq(FOO, HELLO), RangeFilter.class, "RangeConditionFilter(Foo greater than \"Hello\")"); - inverse(FilterComparison.leq(HELLO, FOO), RangeConditionFilter.class, + inverse(FilterComparison.leq(HELLO, FOO), RangeFilter.class, "RangeConditionFilter(Foo less than \"Hello\")"); inverse(FilterComparison.leq(FOO, BAR), ConditionFilter.class, "Foo > Bar"); } diff --git a/server/src/main/java/io/deephaven/server/table/ops/filter/FilterFactory.java b/server/src/main/java/io/deephaven/server/table/ops/filter/FilterFactory.java index 7b5edc29c9a..4ad44c629ca 100644 --- a/server/src/main/java/io/deephaven/server/table/ops/filter/FilterFactory.java +++ b/server/src/main/java/io/deephaven/server/table/ops/filter/FilterFactory.java @@ -11,7 +11,7 @@ import io.deephaven.engine.table.impl.select.DisjunctiveFilter; import io.deephaven.engine.table.impl.select.FormulaParserConfiguration; import io.deephaven.engine.table.impl.select.MatchFilter; -import io.deephaven.engine.table.impl.select.RangeConditionFilter; +import io.deephaven.engine.table.impl.select.RangeFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterFactory; import io.deephaven.engine.table.impl.select.WhereNoneFilter; @@ -145,7 +145,7 @@ private WhereFilter generateNumericConditionFilter(CompareCondition.CompareOpera default: throw new IllegalStateException("Range filter can't handle literal type " + value.getValueCase()); } - return new RangeConditionFilter(columName, rangeCondition(operation, invert), valueString, null, + return new RangeFilter(columName, rangeCondition(operation, invert), valueString, null, FormulaParserConfiguration.parser); } From 286f51b9f9595da2a8d6bf15b16ee7f4ceccdf53 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 11 Jun 2024 11:17:29 -0600 Subject: [PATCH 09/44] f --- .../deephaven/engine/table/impl/select/RangeFilter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 74cf6cb9f3b..1ebd4589ce2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -68,7 +68,7 @@ public RangeFilter(String columnName, Condition condition, String value) { * @param parserConfiguration the parser configuration to use */ public RangeFilter(String columnName, Condition condition, String value, String expression, - FormulaParserConfiguration parserConfiguration) { + FormulaParserConfiguration parserConfiguration) { this(columnName, condition, value, expression, null, parserConfiguration); } @@ -82,13 +82,13 @@ public RangeFilter(String columnName, Condition condition, String value, String * @param parserConfiguration the parser configuration to useyy */ public RangeFilter(String columnName, String conditionString, String value, String expression, - FormulaParserConfiguration parserConfiguration) { + FormulaParserConfiguration parserConfiguration) { this(columnName, conditionFromString(conditionString), value, expression, parserConfiguration); } // Used for copy method private RangeFilter(String columnName, Condition condition, String value, String expression, - WhereFilter filter, FormulaParserConfiguration parserConfiguration) { + WhereFilter filter, FormulaParserConfiguration parserConfiguration) { Assert.eqTrue(conditionSupported(condition), condition + " is not supported by RangeConditionFilter"); this.columnName = columnName; this.condition = condition; @@ -288,7 +288,7 @@ public WritableRowSet filterInverse( @Override public boolean isSimpleFilter() { - return true; + return filter.isSimpleFilter(); } @Override From da8bcdf15f2cc34c80212db8027d20b6b27eeab9 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 11 Jun 2024 11:23:51 -0600 Subject: [PATCH 10/44] Add failover to MatchFilter --- .../engine/table/impl/select/MatchFilter.java | 109 ++++++++++++++---- .../table/impl/select/WhereFilterFactory.java | 3 + 2 files changed, 89 insertions(+), 23 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index c1530e31892..8717d5920a5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -3,6 +3,7 @@ // package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -45,11 +46,19 @@ static MatchFilter ofLiterals( Collection literals, boolean inverted) { return new MatchFilter( + null, null, null, inverted ? MatchType.Inverted : MatchType.Regular, columnName, literals.stream().map(AsObject::of).toArray()); } + /** The expression to use for this filter should initialization fail. */ + private final String expression; + /** The failover filter to use if this filter fails to initialize. */ + private WhereFilter failoverFilter; + /** The parser configuration to use for the failover filter. */ + private final FormulaParserConfiguration parserConfiguration; + @NotNull private final String columnName; private Object[] values; @@ -83,7 +92,7 @@ public MatchFilter( @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final Object... values) { - this(CaseSensitivity.MatchCase, matchType, columnName, null, values); + this(null, null, null, CaseSensitivity.MatchCase, matchType, columnName, null, values); } /** @@ -94,30 +103,61 @@ public MatchFilter( public MatchFilter( @NotNull final String columnName, @NotNull final Object... values) { - this(CaseSensitivity.IgnoreCase, MatchType.Regular, columnName, null, values); + this(null, null, null, CaseSensitivity.IgnoreCase, MatchType.Regular, columnName, null, values); } public MatchFilter( @NotNull final CaseSensitivity sensitivity, + @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final String... strValues) { - this(sensitivity, MatchType.Regular, columnName, strValues, null); + this(null, null, null, sensitivity, matchType, columnName, strValues, null); } public MatchFilter( + @Nullable final String expression, + @Nullable final FormulaParserConfiguration parserConfiguration, + @NotNull final CaseSensitivity sensitivity, + @NotNull final MatchType matchType, + @NotNull final String columnName, + @NotNull final String... strValues) { + this(expression, parserConfiguration, null, sensitivity, matchType, columnName, strValues, null); + } + + private MatchFilter( + @Nullable final String expression, + @Nullable final FormulaParserConfiguration parserConfiguration, + @Nullable final WhereFilter failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final String... strValues) { - this(sensitivity, matchType, columnName, strValues, null); + this(expression, parserConfiguration, failoverFilter, sensitivity, matchType, columnName, strValues, null); + } + + private MatchFilter( + @Nullable final String expression, + @Nullable final FormulaParserConfiguration parserConfiguration, + @Nullable final WhereFilter failoverFilter, + @NotNull final MatchType matchType, + @NotNull final String columnName, + @NotNull final Object... values) { + this(expression, parserConfiguration, failoverFilter, CaseSensitivity.MatchCase, matchType, columnName, null, + values); } private MatchFilter( + @Nullable final String expression, + @Nullable final FormulaParserConfiguration parserConfiguration, + @Nullable final WhereFilter failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @Nullable String[] strValues, @Nullable final Object[] values) { + this.expression = expression; + this.parserConfiguration = parserConfiguration; + this.failoverFilter = failoverFilter; this.caseInsensitive = sensitivity == CaseSensitivity.IgnoreCase; this.invertMatch = (matchType == MatchType.Inverted); this.columnName = columnName; @@ -127,9 +167,10 @@ private MatchFilter( public MatchFilter renameFilter(String newName) { if (strValues == null) { - return new MatchFilter(getMatchType(), newName, values); + return new MatchFilter(expression, parserConfiguration, failoverFilter, getMatchType(), newName, values); } else { return new MatchFilter( + expression, parserConfiguration, failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), newName, strValues); } @@ -173,23 +214,31 @@ public synchronized void init( if (initialized) { return; } - ColumnDefinition column = tableDefinition.getColumn(columnName); - if (column == null) { - throw new RuntimeException("Column \"" + columnName - + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); - } - if (strValues == null) { - initialized = true; - return; - } - final List valueList = new ArrayList<>(); - final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); - final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType()); - for (String strValue : strValues) { - convertor.convertValue(column, strValue, queryScopeVariables, valueList::add); + try { + ColumnDefinition column = tableDefinition.getColumn(columnName); + if (column == null) { + throw new RuntimeException("Column \"" + columnName + + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); + } + if (strValues == null) { + initialized = true; + return; + } + final List valueList = new ArrayList<>(); + final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); + final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType()); + for (String strValue : strValues) { + convertor.convertValue(column, strValue, queryScopeVariables, valueList::add); + } + // values = (Object[])ArrayTypeUtils.toArray(valueList, TypeUtils.getBoxedType(theColumn.getDataType())); + values = valueList.toArray(); + } catch (final RuntimeException err) { + if (expression != null) { + failoverFilter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + } else { + throw new UncheckedDeephavenException("MatchFilter had an unexpected error", err); + } } - // values = (Object[])ArrayTypeUtils.toArray(valueList, TypeUtils.getBoxedType(theColumn.getDataType())); - values = valueList.toArray(); initialized = true; } @@ -231,6 +280,10 @@ public Stream getDependencyStream() { @Override public WritableRowSet filter( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { + if (failoverFilter != null) { + return failoverFilter.filter(selection, fullSet, table, usePrev); + } + final ColumnSource columnSource = table.getColumnSource(columnName); return columnSource.match(invertMatch, usePrev, caseInsensitive, dataIndex, selection, values); } @@ -239,12 +292,20 @@ public WritableRowSet filter( @Override public WritableRowSet filterInverse( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { + if (failoverFilter != null) { + return failoverFilter.filterInverse(selection, fullSet, table, usePrev); + } + final ColumnSource columnSource = table.getColumnSource(columnName); return columnSource.match(!invertMatch, usePrev, caseInsensitive, dataIndex, selection, values); } @Override public boolean isSimpleFilter() { + if (failoverFilter != null) { + return failoverFilter.isSimpleFilter(); + } + return true; } @@ -619,10 +680,12 @@ public boolean canMemoize() { public WhereFilter copy() { final MatchFilter copy; if (strValues != null) { - copy = new MatchFilter(caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, + copy = new MatchFilter( + expression, parserConfiguration, failoverFilter, + caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), columnName, strValues); } else { - copy = new MatchFilter(getMatchType(), columnName, values); + copy = new MatchFilter(expression, parserConfiguration, failoverFilter, getMatchType(), columnName, values); } if (initialized) { copy.initialized = true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index ce4d0b79ae8..ce481df3376 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -131,10 +131,12 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a final boolean icase = matcher.group(2) != null; final boolean inverted = matcher.group(3) != null; final String[] values = new SplitIgnoreQuotes().split(matcher.group(4), ','); + final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( + expression, parserConfiguration, icase ? MatchFilter.CaseSensitivity.IgnoreCase : MatchFilter.CaseSensitivity.MatchCase, inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, columnName, @@ -208,6 +210,7 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( + expression, parserConfiguration, CaseSensitivity.MatchCase, mirrored ? MatchType.Inverted : MatchType.Regular, columnName, From 9b3fe23088148bb06910ccc5082fa51e5a67e7ff Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 11 Jun 2024 14:12:00 -0600 Subject: [PATCH 11/44] reduce regex to single pattern --- .../table/impl/select/WhereFilterFactory.java | 80 ++++--------------- 1 file changed, 16 insertions(+), 64 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index ce481df3376..49a30658235 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -49,79 +49,30 @@ public class WhereFilterFactory { private static final ExpressionParser parser = new ExpressionParser<>(); static { - // == - // = - // != - // < - // <= - // > - // >= + // Each side may fit: (||) + // Supported ops: ==, =, !=, <, <=, >, >= parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + LITERAL_PTRN + ")" + END_PTRN) { + START_PTRN + "(?:(" + ID_PTRN + ")|(" + LITERAL_PTRN + "))\\s*((?:=|!|<|>)=?)\\s*(?:(" + ID_PTRN + ")|(" + LITERAL_PTRN + "))" + END_PTRN) { @Override public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { - final String columnName = matcher.group(1); - final String op = matcher.group(2); - final String value = matcher.group(3); - final boolean mirrored = false; + // LITERAL_PTRN has 5 groups; mostly non-capturing + final boolean leftIsId = matcher.group(1) != null; + final boolean rightIsId = matcher.group(8) != null; - return getWhereFilterOneSideColumn( - expression, (FormulaParserConfiguration) args[0], columnName, op, value, mirrored); - } - }); + if (!leftIsId && !rightIsId) { + return ConditionFilter.createConditionFilter(expression, (FormulaParserConfiguration) args[0]); + } + final boolean mirrored = !leftIsId; - // == - // = - // != - // < - // <= - // > - // >= - parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + LITERAL_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { - @Override - public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { - final String value = matcher.group(1); - final String op = matcher.group(6); - final String columnName = matcher.group(7); - final boolean mirrored = true; + final String columnName = leftIsId ? matcher.group(1) : matcher.group(8); + final String op = matcher.group(7); + final String value = leftIsId ? rightIsId ? matcher.group(8) : matcher.group(9) : matcher.group(2); return getWhereFilterOneSideColumn( expression, (FormulaParserConfiguration) args[0], columnName, op, value, mirrored); } }); - // == - // = - // != - // < - // <= - // > - // >= - parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(" + ID_PTRN + ")\\s*((?:=|!|<|>)=?)\\s*(" + ID_PTRN + ")" + END_PTRN) { - @Override - public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { - final String columnName = matcher.group(1); - final String op = matcher.group(2); - final String paramName = matcher.group(3); - final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; - - boolean mirrored = false; - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - if (!queryScope.hasParamName(paramName)) { - if (queryScope.hasParamName(columnName)) { - mirrored = true; - } else { - return ConditionFilter.createConditionFilter(expression, parserConfiguration); - } - } - - return getWhereFilterOneSideColumn( - expression, parserConfiguration, columnName, op, paramName, mirrored); - } - }); - // [icase] [not] in , , ... , parser.registerFactory(new AbstractExpressionFactory<>("(?s)" + START_PTRN + "(" + ID_PTRN + ")\\s+(" + ICASE + "\\s+)?(" + NOT + "\\s+)?" + IN + "\\s+(.+?)" + END_PTRN) { @@ -202,9 +153,10 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a return ConditionFilter.createConditionFilter(expression, parserConfiguration); } + boolean inverted = false; switch (op) { case "!=": - mirrored = !mirrored; + inverted = !inverted; case "=": case "==": log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) @@ -212,7 +164,7 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a return new MatchFilter( expression, parserConfiguration, CaseSensitivity.MatchCase, - mirrored ? MatchType.Inverted : MatchType.Regular, + inverted ? MatchType.Inverted : MatchType.Regular, columnName, value); From 7488c97a690ced4c2732f71a6150534a5665a5bc Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 11 Jun 2024 21:01:05 -0600 Subject: [PATCH 12/44] fix spotless --- .../deephaven/engine/table/impl/select/WhereFilterFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 49a30658235..0c240b7f4c9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -52,7 +52,8 @@ public class WhereFilterFactory { // Each side may fit: (||) // Supported ops: ==, =, !=, <, <=, >, >= parser.registerFactory(new AbstractExpressionFactory<>( - START_PTRN + "(?:(" + ID_PTRN + ")|(" + LITERAL_PTRN + "))\\s*((?:=|!|<|>)=?)\\s*(?:(" + ID_PTRN + ")|(" + LITERAL_PTRN + "))" + END_PTRN) { + START_PTRN + "(?:(" + ID_PTRN + ")|(" + LITERAL_PTRN + "))\\s*((?:=|!|<|>)=?)\\s*(?:(" + ID_PTRN + ")|(" + + LITERAL_PTRN + "))" + END_PTRN) { @Override public WhereFilter getExpression(String expression, Matcher matcher, Object... args) { // LITERAL_PTRN has 5 groups; mostly non-capturing From df7e0b1249962c4ed46921edd2c72b6555455a7c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 12 Jun 2024 15:24:33 -0600 Subject: [PATCH 13/44] bugfix --- .../java/io/deephaven/engine/table/impl/select/MatchFilter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 8717d5920a5..e68e959002d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -235,6 +235,7 @@ public synchronized void init( } catch (final RuntimeException err) { if (expression != null) { failoverFilter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + failoverFilter.init(tableDefinition, compilationProcessor); } else { throw new UncheckedDeephavenException("MatchFilter had an unexpected error", err); } From 18a8ab76d7e7ff289160f89542dcda668b649f91 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 12 Jun 2024 16:07:49 -0600 Subject: [PATCH 14/44] where filter test fixes --- .../engine/table/impl/select/RangeFilter.java | 2 +- .../table/impl/select/WhereFilterTest.java | 92 +++++++++---------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 1ebd4589ce2..f4e1281efa2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -301,6 +301,6 @@ public WhereFilter copy() { @Override public String toString() { - return "RangeConditionFilter(" + columnName + " " + condition.description + " " + value + ")"; + return "RangeFilter(" + columnName + " " + condition.description + " " + value + ")"; } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java index a1aaec275da..f1900291d51 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterTest.java @@ -73,13 +73,13 @@ public void testEq() { regular(FilterComparison.eq(V42, FOO), MatchFilter.class, "Foo in [42]"); regular(FilterComparison.eq(FOO, HELLO), MatchFilter.class, "Foo in [Hello]"); regular(FilterComparison.eq(HELLO, FOO), MatchFilter.class, "Foo in [Hello]"); - regular(FilterComparison.eq(FOO, BAR), ConditionFilter.class, "Foo == Bar"); + regular(FilterComparison.eq(FOO, BAR), MatchFilter.class, "Foo in [Bar]"); inverse(FilterComparison.eq(FOO, V42), MatchFilter.class, "Foo not in [42]"); inverse(FilterComparison.eq(V42, FOO), MatchFilter.class, "Foo not in [42]"); inverse(FilterComparison.eq(FOO, HELLO), MatchFilter.class, "Foo not in [Hello]"); inverse(FilterComparison.eq(HELLO, FOO), MatchFilter.class, "Foo not in [Hello]"); - inverse(FilterComparison.eq(FOO, BAR), ConditionFilter.class, "Foo != Bar"); + inverse(FilterComparison.eq(FOO, BAR), MatchFilter.class, "Foo not in [Bar]"); } public void testNeq() { @@ -87,101 +87,101 @@ public void testNeq() { regular(FilterComparison.neq(V42, FOO), MatchFilter.class, "Foo not in [42]"); regular(FilterComparison.neq(FOO, HELLO), MatchFilter.class, "Foo not in [Hello]"); regular(FilterComparison.neq(HELLO, FOO), MatchFilter.class, "Foo not in [Hello]"); - regular(FilterComparison.neq(FOO, BAR), ConditionFilter.class, "Foo != Bar"); + regular(FilterComparison.neq(FOO, BAR), MatchFilter.class, "Foo not in [Bar]"); inverse(FilterComparison.neq(FOO, V42), MatchFilter.class, "Foo in [42]"); inverse(FilterComparison.neq(V42, FOO), MatchFilter.class, "Foo in [42]"); inverse(FilterComparison.neq(FOO, HELLO), MatchFilter.class, "Foo in [Hello]"); inverse(FilterComparison.neq(HELLO, FOO), MatchFilter.class, "Foo in [Hello]"); - inverse(FilterComparison.neq(FOO, BAR), ConditionFilter.class, "Foo == Bar"); + inverse(FilterComparison.neq(FOO, BAR), MatchFilter.class, "Foo in [Bar]"); } public void testGt() { regular(FilterComparison.gt(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo greater than 42)"); + "RangeFilter(Foo greater than 42)"); regular(FilterComparison.gt(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than 42)"); + "RangeFilter(Foo less than 42)"); regular(FilterComparison.gt(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo greater than \"Hello\")"); + "RangeFilter(Foo greater than \"Hello\")"); regular(FilterComparison.gt(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than \"Hello\")"); - regular(FilterComparison.gt(FOO, BAR), ConditionFilter.class, "Foo > Bar"); + "RangeFilter(Foo less than \"Hello\")"); + regular(FilterComparison.gt(FOO, BAR), RangeFilter.class, "RangeFilter(Foo greater than Bar)"); inverse(FilterComparison.gt(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to 42)"); + "RangeFilter(Foo less than or equal to 42)"); inverse(FilterComparison.gt(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to 42)"); + "RangeFilter(Foo greater than or equal to 42)"); inverse(FilterComparison.gt(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to \"Hello\")"); + "RangeFilter(Foo less than or equal to \"Hello\")"); inverse(FilterComparison.gt(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); - inverse(FilterComparison.gt(FOO, BAR), ConditionFilter.class, "Foo <= Bar"); + "RangeFilter(Foo greater than or equal to \"Hello\")"); + inverse(FilterComparison.gt(FOO, BAR), RangeFilter.class, "RangeFilter(Foo less than or equal to Bar)"); } public void testGte() { regular(FilterComparison.geq(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to 42)"); + "RangeFilter(Foo greater than or equal to 42)"); regular(FilterComparison.geq(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to 42)"); + "RangeFilter(Foo less than or equal to 42)"); regular(FilterComparison.geq(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); + "RangeFilter(Foo greater than or equal to \"Hello\")"); regular(FilterComparison.geq(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to \"Hello\")"); - regular(FilterComparison.geq(FOO, BAR), ConditionFilter.class, "Foo >= Bar"); + "RangeFilter(Foo less than or equal to \"Hello\")"); + regular(FilterComparison.geq(FOO, BAR), RangeFilter.class, "RangeFilter(Foo greater than or equal to Bar)"); inverse(FilterComparison.geq(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo less than 42)"); + "RangeFilter(Foo less than 42)"); inverse(FilterComparison.geq(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than 42)"); + "RangeFilter(Foo greater than 42)"); inverse(FilterComparison.geq(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo less than \"Hello\")"); + "RangeFilter(Foo less than \"Hello\")"); inverse(FilterComparison.geq(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than \"Hello\")"); - inverse(FilterComparison.geq(FOO, BAR), ConditionFilter.class, "Foo < Bar"); + "RangeFilter(Foo greater than \"Hello\")"); + inverse(FilterComparison.geq(FOO, BAR), RangeFilter.class, "RangeFilter(Foo less than Bar)"); } public void testLt() { regular(FilterComparison.lt(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo less than 42)"); + "RangeFilter(Foo less than 42)"); regular(FilterComparison.lt(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than 42)"); + "RangeFilter(Foo greater than 42)"); regular(FilterComparison.lt(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo less than \"Hello\")"); + "RangeFilter(Foo less than \"Hello\")"); regular(FilterComparison.lt(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than \"Hello\")"); - regular(FilterComparison.lt(FOO, BAR), ConditionFilter.class, "Foo < Bar"); + "RangeFilter(Foo greater than \"Hello\")"); + regular(FilterComparison.lt(FOO, BAR), RangeFilter.class, "RangeFilter(Foo less than Bar)"); inverse(FilterComparison.lt(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to 42)"); + "RangeFilter(Foo greater than or equal to 42)"); inverse(FilterComparison.lt(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to 42)"); + "RangeFilter(Foo less than or equal to 42)"); inverse(FilterComparison.lt(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); + "RangeFilter(Foo greater than or equal to \"Hello\")"); inverse(FilterComparison.lt(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to \"Hello\")"); - inverse(FilterComparison.lt(FOO, BAR), ConditionFilter.class, "Foo >= Bar"); + "RangeFilter(Foo less than or equal to \"Hello\")"); + inverse(FilterComparison.lt(FOO, BAR), RangeFilter.class, "RangeFilter(Foo greater than or equal to Bar)"); } public void testLte() { regular(FilterComparison.leq(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to 42)"); + "RangeFilter(Foo less than or equal to 42)"); regular(FilterComparison.leq(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to 42)"); + "RangeFilter(Foo greater than or equal to 42)"); regular(FilterComparison.leq(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo less than or equal to \"Hello\")"); + "RangeFilter(Foo less than or equal to \"Hello\")"); regular(FilterComparison.leq(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo greater than or equal to \"Hello\")"); - regular(FilterComparison.leq(FOO, BAR), ConditionFilter.class, "Foo <= Bar"); + "RangeFilter(Foo greater than or equal to \"Hello\")"); + regular(FilterComparison.leq(FOO, BAR), RangeFilter.class, "RangeFilter(Foo less than or equal to Bar)"); inverse(FilterComparison.leq(FOO, V42), RangeFilter.class, - "RangeConditionFilter(Foo greater than 42)"); + "RangeFilter(Foo greater than 42)"); inverse(FilterComparison.leq(V42, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than 42)"); + "RangeFilter(Foo less than 42)"); inverse(FilterComparison.leq(FOO, HELLO), RangeFilter.class, - "RangeConditionFilter(Foo greater than \"Hello\")"); + "RangeFilter(Foo greater than \"Hello\")"); inverse(FilterComparison.leq(HELLO, FOO), RangeFilter.class, - "RangeConditionFilter(Foo less than \"Hello\")"); - inverse(FilterComparison.leq(FOO, BAR), ConditionFilter.class, "Foo > Bar"); + "RangeFilter(Foo less than \"Hello\")"); + inverse(FilterComparison.leq(FOO, BAR), RangeFilter.class, "RangeFilter(Foo greater than Bar)"); } public void testFunction() { @@ -303,8 +303,8 @@ public void testInLiteralsDifferentTypes() { public void testInSingleNotLiteral() { final FilterIn in = FilterIn.of(FOO, BAR); - regular(in, ConditionFilter.class, "Foo == Bar"); - inverse(in, ConditionFilter.class, "Foo != Bar"); + regular(in, MatchFilter.class, "Foo in [Bar]"); + inverse(in, MatchFilter.class, "Foo not in [Bar]"); } From 804d950adbbb8385a99831467676f6a7f2ab8940 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Wed, 12 Jun 2024 16:16:05 -0600 Subject: [PATCH 15/44] where filter factory test --- .../engine/table/impl/select/WhereFilterFactoryTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java index 453479692cf..b870b8adc25 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java @@ -59,9 +59,10 @@ public void testInComplex() { try { filter.init(tableDef); - } catch (IllegalArgumentException e) { + } catch (FormulaCompilationException e) { assertEquals(e.getMessage(), - "Failed to convert literal value <1> for column \"Maturity\" of type java.lang.String"); + "Formula compilation error for: Maturity in amExpiry,1 , \"AGAIN\" "); + } } From 46bf8316f2d0603309abe3fb6af236bfcaea70b2 Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 18 Jun 2024 11:09:41 -0600 Subject: [PATCH 16/44] Update engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java Co-authored-by: Ryan Caudy --- .../engine/table/impl/select/MatchFilter.java | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index e68e959002d..9092110e586 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -340,32 +340,31 @@ final boolean convertValue( @NotNull final String strValue, @NotNull final Map queryScopeVariables, @NotNull final Consumer valueConsumer) { - if (queryScopeVariables.containsKey(strValue)) { - Object paramValue = queryScopeVariables.get(strValue); - if (paramValue != null && paramValue.getClass().isArray()) { - ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); - for (int ai = 0; ai < accessor.length(); ++ai) { - valueConsumer.accept(convertParamValue(accessor.get(ai))); - } - return true; - } else if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { - for (final Object paramValueMember : (Collection) paramValue) { - valueConsumer.accept(convertParamValue(paramValueMember)); - } - return true; - } else { - valueConsumer.accept(convertParamValue(paramValue)); - } - } else { - try { - valueConsumer.accept(convertStringLiteral(strValue)); - } catch (Throwable t) { - throw new IllegalArgumentException("Failed to convert literal value <" + strValue + - "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); - } + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); + if (paramValue != null && paramValue.getClass().isArray()) { + ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); + for (int ai = 0; ai < accessor.length(); ++ai) { + valueConsumer.accept(convertParamValue(accessor.get(ai))); } - return false; + return true; + } + if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { + for (final Object paramValueMember : (Collection) paramValue) { + valueConsumer.accept(convertParamValue(paramValueMember)); + } + return true; } + valueConsumer.accept(convertParamValue(paramValue)); + return false; + } + try { + valueConsumer.accept(convertStringLiteral(strValue)); + } catch (Throwable t) { + throw new IllegalArgumentException("Failed to convert literal value <" + strValue + + "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); + } + return false; } public static class ColumnTypeConvertorFactory { From 136c718a21cfd5cb4315d4c47c85b0b16fa68ddb Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 18 Jun 2024 11:09:52 -0600 Subject: [PATCH 17/44] Update engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java Co-authored-by: Ryan Caudy --- .../deephaven/engine/table/impl/select/WhereFilterFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 0c240b7f4c9..cb3e4801090 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -178,7 +178,7 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a final String dir = op.substring(0, 1); op = op.replaceFirst(dir, dir.equals("<") ? ">" : "<"); } - log.debug().append("WhereFilterFactory creating RangeConditionFilter for expression: ") + log.debug().append("WhereFilterFactory creating RangeFilter for expression: ") .append(expression).endl(); return new RangeFilter(columnName, op, value, expression, parserConfiguration); } catch (Exception e) { From e0951feb396093e779cd6b778e17fc481944d96c Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 18 Jun 2024 11:09:59 -0600 Subject: [PATCH 18/44] Update engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java Co-authored-by: Ryan Caudy --- .../deephaven/engine/table/impl/select/WhereFilterFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index cb3e4801090..58b25062fcf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -157,7 +157,7 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a boolean inverted = false; switch (op) { case "!=": - inverted = !inverted; + inverted = true; case "=": case "==": log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) From 2795e172d11a4c5f58a01f5ed8aa978a88a5fb70 Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 18 Jun 2024 11:14:36 -0600 Subject: [PATCH 19/44] Update engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java Co-authored-by: Ryan Caudy --- .../java/io/deephaven/engine/table/impl/select/RangeFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index f4e1281efa2..70fd2319655 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -79,7 +79,7 @@ public RangeFilter(String columnName, Condition condition, String value, String * @param conditionString the String representation of a condition for filtering * @param value a String representation of the numeric filter value * @param expression the original expression prior to being parsed - * @param parserConfiguration the parser configuration to useyy + * @param parserConfiguration the parser configuration to use */ public RangeFilter(String columnName, String conditionString, String value, String expression, FormulaParserConfiguration parserConfiguration) { From d098d219676c9145b29240c3b30774abd4cebd0a Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Tue, 18 Jun 2024 11:14:50 -0600 Subject: [PATCH 20/44] Update engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java Co-authored-by: Ryan Caudy --- .../java/io/deephaven/engine/table/impl/select/RangeFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 70fd2319655..f93562cba45 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -165,7 +165,7 @@ public void init( boolean wasAnArrayType = convertor.convertValue( def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); if (wasAnArrayType) { - throw new IllegalArgumentException("RangeConditionFilter does not support array types for column " + throw new IllegalArgumentException("RangeFilter does not support array types for column " + columnName + " with value <" + value + ">"); } } catch (final RuntimeException err) { From 6982d747ec932cf33dff90497628ce17a931c42e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 18 Jun 2024 11:15:00 -0600 Subject: [PATCH 21/44] Ryan's feedback --- .../util/datastructures/CachingSupplier.java | 10 ++ .../engine/table/impl/select/MatchFilter.java | 156 ++++++++---------- .../table/impl/select/WhereFilterFactory.java | 27 +-- .../impl/select/WhereFilterFactoryTest.java | 4 +- .../java/io/deephaven/time/DateTimeUtils.java | 70 ++++++++ 5 files changed, 158 insertions(+), 109 deletions(-) diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java index 0a0880f7896..d36719b1ad0 100644 --- a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -49,4 +49,14 @@ public OUTPUT_TYPE get() { } return cachedResult; } + + public OUTPUT_TYPE getIfCached() { + if (hasCachedResult) { // force a volatile read + if (errorResult != null) { + throw errorResult; + } + return cachedResult; + } + return null; + } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 9092110e586..97d9533a6c5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -3,7 +3,6 @@ // package io.deephaven.engine.table.impl.select; -import io.deephaven.UncheckedDeephavenException; import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; import io.deephaven.engine.liveness.LivenessScopeStack; @@ -21,6 +20,7 @@ import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.SafeCloseable; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.util.type.ArrayTypeUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -46,18 +46,13 @@ static MatchFilter ofLiterals( Collection literals, boolean inverted) { return new MatchFilter( - null, null, null, inverted ? MatchType.Inverted : MatchType.Regular, columnName, literals.stream().map(AsObject::of).toArray()); } - /** The expression to use for this filter should initialization fail. */ - private final String expression; - /** The failover filter to use if this filter fails to initialize. */ - private WhereFilter failoverFilter; - /** The parser configuration to use for the failover filter. */ - private final FormulaParserConfiguration parserConfiguration; + /** A fail-over where filter supplier should the match filter initialization fail. */ + private final CachingSupplier failoverFilter; @NotNull private final String columnName; @@ -88,11 +83,13 @@ public enum CaseSensitivity { MatchCase, IgnoreCase } + // TODO NATE NOCOMMIT: USE BUILDER TO CREATE MATCH FILTER?? + public MatchFilter( @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final Object... values) { - this(null, null, null, CaseSensitivity.MatchCase, matchType, columnName, null, values); + this(null, CaseSensitivity.MatchCase, matchType, columnName, null, values); } /** @@ -103,7 +100,7 @@ public MatchFilter( public MatchFilter( @NotNull final String columnName, @NotNull final Object... values) { - this(null, null, null, CaseSensitivity.IgnoreCase, MatchType.Regular, columnName, null, values); + this(null, CaseSensitivity.IgnoreCase, MatchType.Regular, columnName, null, values); } public MatchFilter( @@ -111,52 +108,25 @@ public MatchFilter( @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final String... strValues) { - this(null, null, null, sensitivity, matchType, columnName, strValues, null); + this(null, sensitivity, matchType, columnName, strValues, null); } public MatchFilter( - @Nullable final String expression, - @Nullable final FormulaParserConfiguration parserConfiguration, - @NotNull final CaseSensitivity sensitivity, - @NotNull final MatchType matchType, - @NotNull final String columnName, - @NotNull final String... strValues) { - this(expression, parserConfiguration, null, sensitivity, matchType, columnName, strValues, null); - } - - private MatchFilter( - @Nullable final String expression, - @Nullable final FormulaParserConfiguration parserConfiguration, - @Nullable final WhereFilter failoverFilter, + @Nullable final CachingSupplier failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @NotNull final String... strValues) { - this(expression, parserConfiguration, failoverFilter, sensitivity, matchType, columnName, strValues, null); - } - - private MatchFilter( - @Nullable final String expression, - @Nullable final FormulaParserConfiguration parserConfiguration, - @Nullable final WhereFilter failoverFilter, - @NotNull final MatchType matchType, - @NotNull final String columnName, - @NotNull final Object... values) { - this(expression, parserConfiguration, failoverFilter, CaseSensitivity.MatchCase, matchType, columnName, null, - values); + this(failoverFilter, sensitivity, matchType, columnName, strValues, null); } private MatchFilter( - @Nullable final String expression, - @Nullable final FormulaParserConfiguration parserConfiguration, - @Nullable final WhereFilter failoverFilter, + @Nullable final CachingSupplier failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @Nullable String[] strValues, @Nullable final Object[] values) { - this.expression = expression; - this.parserConfiguration = parserConfiguration; this.failoverFilter = failoverFilter; this.caseInsensitive = sensitivity == CaseSensitivity.IgnoreCase; this.invertMatch = (matchType == MatchType.Inverted); @@ -167,12 +137,11 @@ private MatchFilter( public MatchFilter renameFilter(String newName) { if (strValues == null) { - return new MatchFilter(expression, parserConfiguration, failoverFilter, getMatchType(), newName, values); + return new MatchFilter(getMatchType(), newName, values); } else { return new MatchFilter( - expression, parserConfiguration, failoverFilter, - caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, - getMatchType(), newName, strValues); + failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, + getMatchType(), newName, strValues, null); } } @@ -230,14 +199,15 @@ public synchronized void init( for (String strValue : strValues) { convertor.convertValue(column, strValue, queryScopeVariables, valueList::add); } - // values = (Object[])ArrayTypeUtils.toArray(valueList, TypeUtils.getBoxedType(theColumn.getDataType())); values = valueList.toArray(); } catch (final RuntimeException err) { - if (expression != null) { - failoverFilter = ConditionFilter.createConditionFilter(expression, parserConfiguration); - failoverFilter.init(tableDefinition, compilationProcessor); - } else { - throw new UncheckedDeephavenException("MatchFilter had an unexpected error", err); + if (failoverFilter == null) { + throw err; + } + try { + failoverFilter.get().init(tableDefinition, compilationProcessor); + } catch (final RuntimeException ignored) { + throw err; } } initialized = true; @@ -281,8 +251,9 @@ public Stream getDependencyStream() { @Override public WritableRowSet filter( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { - if (failoverFilter != null) { - return failoverFilter.filter(selection, fullSet, table, usePrev); + final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + if (failover != null) { + return failover.filter(selection, fullSet, table, usePrev); } final ColumnSource columnSource = table.getColumnSource(columnName); @@ -293,8 +264,9 @@ public WritableRowSet filter( @Override public WritableRowSet filterInverse( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { - if (failoverFilter != null) { - return failoverFilter.filterInverse(selection, fullSet, table, usePrev); + final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + if (failover != null) { + return failover.filterInverse(selection, fullSet, table, usePrev); } final ColumnSource columnSource = table.getColumnSource(columnName); @@ -303,8 +275,9 @@ public WritableRowSet filterInverse( @Override public boolean isSimpleFilter() { - if (failoverFilter != null) { - return failoverFilter.isSimpleFilter(); + final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + if (failover != null) { + return failover.isSimpleFilter(); } return true; @@ -328,11 +301,11 @@ Object convertParamValue(Object paramValue) { /** * Convert the string value to the appropriate type for the column. - * - * @param column the column definition - * @param strValue the string value to convert + * + * @param column the column definition + * @param strValue the string value to convert * @param queryScopeVariables the query scope variables - * @param valueConsumer the consumer for the converted value + * @param valueConsumer the consumer for the converted value * @return whether the value was an array or collection */ final boolean convertValue( @@ -340,33 +313,34 @@ final boolean convertValue( @NotNull final String strValue, @NotNull final Map queryScopeVariables, @NotNull final Consumer valueConsumer) { - if (queryScopeVariables.containsKey(strValue)) { - Object paramValue = queryScopeVariables.get(strValue); - if (paramValue != null && paramValue.getClass().isArray()) { - ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); - for (int ai = 0; ai < accessor.length(); ++ai) { - valueConsumer.accept(convertParamValue(accessor.get(ai))); + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); + if (paramValue != null && paramValue.getClass().isArray()) { + ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); + for (int ai = 0; ai < accessor.length(); ++ai) { + valueConsumer.accept(convertParamValue(accessor.get(ai))); + } + return true; + } + if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { + for (final Object paramValueMember : (Collection) paramValue) { + valueConsumer.accept(convertParamValue(paramValueMember)); + } + return true; + } + valueConsumer.accept(convertParamValue(paramValue)); + return false; } - return true; - } - if (paramValue != null && Collection.class.isAssignableFrom(paramValue.getClass())) { - for (final Object paramValueMember : (Collection) paramValue) { - valueConsumer.accept(convertParamValue(paramValueMember)); + try { + valueConsumer.accept(convertStringLiteral(strValue)); + } catch (Throwable t) { + throw new IllegalArgumentException("Failed to convert literal value <" + strValue + + "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); } - return true; + return false; } - valueConsumer.accept(convertParamValue(paramValue)); - return false; - } - try { - valueConsumer.accept(convertStringLiteral(strValue)); - } catch (Throwable t) { - throw new IllegalArgumentException("Failed to convert literal value <" + strValue + - "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); - } - return false; } - + public static class ColumnTypeConvertorFactory { public static ColumnTypeConvertor getConvertor(final Class cls) { if (cls == byte.class) { @@ -574,7 +548,7 @@ Object convertStringLiteral(String str) { throw new IllegalArgumentException( "LocalDateTime literal not enclosed in single-quotes (\"" + str + "\")"); } - return LocalDateTime.parse(str.substring(1, str.length() - 1)); + return DateTimeUtils.parseLocalDateTime(str.substring(1, str.length() - 1)); } }; } @@ -680,12 +654,16 @@ public boolean canMemoize() { public WhereFilter copy() { final MatchFilter copy; if (strValues != null) { + final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + if (failover != null) { + return failover.copy(); + } + copy = new MatchFilter( - expression, parserConfiguration, failoverFilter, - caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, - getMatchType(), columnName, strValues); + failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, + getMatchType(), columnName, strValues, null); } else { - copy = new MatchFilter(expression, parserConfiguration, failoverFilter, getMatchType(), columnName, values); + copy = new MatchFilter(getMatchType(), columnName, values); } if (initialized) { copy.initialized = true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 58b25062fcf..b029525c687 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -7,8 +7,6 @@ import io.deephaven.api.filter.FilterPattern; import io.deephaven.api.filter.FilterPattern.Mode; import io.deephaven.base.Pair; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.api.expression.AbstractExpressionFactory; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; @@ -22,6 +20,7 @@ import io.deephaven.io.logger.Logger; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.annotations.VisibleForTesting; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.util.text.SplitIgnoreQuotes; import org.jetbrains.annotations.NotNull; @@ -83,12 +82,10 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a final boolean icase = matcher.group(2) != null; final boolean inverted = matcher.group(3) != null; final String[] values = new SplitIgnoreQuotes().split(matcher.group(4), ','); - final FormulaParserConfiguration parserConfiguration = (FormulaParserConfiguration) args[0]; log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( - expression, parserConfiguration, icase ? MatchFilter.CaseSensitivity.IgnoreCase : MatchFilter.CaseSensitivity.MatchCase, inverted ? MatchFilter.MatchType.Inverted : MatchFilter.MatchType.Regular, columnName, @@ -163,7 +160,8 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( - expression, parserConfiguration, + new CachingSupplier<>( + () -> ConditionFilter.createConditionFilter(expression, parserConfiguration)), CaseSensitivity.MatchCase, inverted ? MatchType.Inverted : MatchType.Regular, columnName, @@ -173,20 +171,13 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a case ">": case "<=": case ">=": - try { - if (mirrored) { - final String dir = op.substring(0, 1); - op = op.replaceFirst(dir, dir.equals("<") ? ">" : "<"); - } - log.debug().append("WhereFilterFactory creating RangeFilter for expression: ") - .append(expression).endl(); - return new RangeFilter(columnName, op, value, expression, parserConfiguration); - } catch (Exception e) { - log.warn().append("WhereFilterFactory could not make RangeFilter for expression: ") - .append(expression).append(" due to ").append(e) - .append(" Creating ConditionFilter instead.").endl(); - return ConditionFilter.createConditionFilter(expression, parserConfiguration); + if (mirrored) { + final String dir = op.substring(0, 1); + op = op.replaceFirst(dir, dir.equals("<") ? ">" : "<"); } + log.debug().append("WhereFilterFactory creating RangeFilter for expression: ") + .append(expression).endl(); + return new RangeFilter(columnName, op, value, expression, parserConfiguration); default: throw new IllegalStateException("Unexpected operator: " + op); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java index b870b8adc25..f5be583ea88 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java @@ -59,9 +59,9 @@ public void testInComplex() { try { filter.init(tableDef); - } catch (FormulaCompilationException e) { + } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), - "Formula compilation error for: Maturity in amExpiry,1 , \"AGAIN\" "); + "Failed to convert literal value <1> for column \"Maturity\" of type java.lang.String"); } } diff --git a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java index 934ebcc6c0a..fe6d549f30f 100644 --- a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java +++ b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java @@ -4373,6 +4373,76 @@ public static Instant parseInstantQuiet(@Nullable final String s) { } } + /** + * Parses the string argument as a {@link LocalDateTime}. + *

    + * Date time strings are formatted according to the ISO 8601 date time format + * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS] TZ} and others. + * + * @param s date time string + * @return a {@link LocalDateTime} represented by the input string + * @throws DateTimeParseException if the string cannot be parsed + * @see DateTimeFormatter#ISO_INSTANT + */ + @ScriptApi + @NotNull + public static LocalDateTime parseLocalDateTime(@NotNull final String s) { + // noinspection ConstantConditions + if (s == null) { + throw new DateTimeParseException("Cannot parse datetime (null): " + s); + } + + try { + return LocalDateTime.parse(s); + } catch (java.time.format.DateTimeParseException e) { + // ignore + } + + try { + final Matcher dtzMatcher = DATE_TZ_PATTERN.matcher(s); + if (dtzMatcher.matches()) { + final String dateString = dtzMatcher.group("date"); + return LocalDate.parse(dateString, FORMATTER_ISO_LOCAL_DATE).atTime(LocalTime.of(0, 0)); + } + + final String dateTimeString; + final int spaceIndex = s.indexOf(' '); + if (spaceIndex != -1) { + dateTimeString = s.substring(0, spaceIndex); + } else { + dateTimeString = s; + } + + return LocalDateTime.parse(dateTimeString, FORMATTER_ISO_LOCAL_DATE_TIME); + } catch (Exception ex) { + throw new DateTimeParseException("Cannot parse local date time: " + s, ex); + } + } + + /** + * Parses the string argument as a {@link LocalDateTime}. + *

    + * Date time strings are formatted according to the ISO 8601 date time format + * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS] TZ} and others. + * + * @param s date time string + * @return a {@link LocalDateTime} represented by the input string, or {@code null} if the string can not be parsed + * @see DateTimeFormatter#ISO_INSTANT + */ + @ScriptApi + @Nullable + public static LocalDateTime parseLocalDateTimeQuiet(@Nullable final String s) { + if (s == null || s.length() <= 1) { + return null; + } + + try { + return parseLocalDateTime(s); + } catch (Exception e) { + return null; + } + } + /** * Parses the string argument as a {@link ZonedDateTime}. *

    From 61fb256a3654f2a9a57a618f823316c9c8a4950c Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 18 Jun 2024 12:38:38 -0600 Subject: [PATCH 22/44] rename --- .../table/impl/select/ByteRangeFilter.java | 2 +- .../table/impl/select/CharRangeFilter.java | 2 +- .../table/impl/select/DoubleRangeFilter.java | 2 +- .../table/impl/select/FloatRangeFilter.java | 2 +- .../table/impl/select/IntRangeFilter.java | 2 +- .../table/impl/select/LongRangeFilter.java | 2 +- .../engine/table/impl/select/RangeFilter.java | 18 +++++++++--------- .../table/impl/select/ShortRangeFilter.java | 2 +- .../table/impl/select/WhereFilterAdapter.java | 4 ++-- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index 5547aa1633a..49bd613eba2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -62,7 +62,7 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, b case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 0041a195cf8..466f58de85a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -58,7 +58,7 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, c case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index bfcaae139b6..edaed1132a9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -74,7 +74,7 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index 9fe1d3d2e98..d06d81f1857 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -70,7 +70,7 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index 4f12b52867c..cbe4daafef5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -62,7 +62,7 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, in case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index 66c7916dc59..353cb457e4c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -62,7 +62,7 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, l case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index f93562cba45..a2731a21266 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -48,7 +48,7 @@ public class RangeFilter extends WhereFilterImpl { private final FormulaParserConfiguration parserConfiguration; /** - * Creates a RangeConditionFilter. + * Creates a RangeFilter. * * @param columnName the column to filter * @param condition the condition for filtering @@ -59,7 +59,7 @@ public RangeFilter(String columnName, Condition condition, String value) { } /** - * Creates a RangeConditionFilter. + * Creates a RangeFilter. * * @param columnName the column to filter * @param condition the condition for filtering @@ -73,7 +73,7 @@ public RangeFilter(String columnName, Condition condition, String value, String } /** - * Creates a RangeConditionFilter. + * Creates a RangeFilter. * * @param columnName the column to filter * @param conditionString the String representation of a condition for filtering @@ -89,7 +89,7 @@ public RangeFilter(String columnName, String conditionString, String value, Stri // Used for copy method private RangeFilter(String columnName, Condition condition, String value, String expression, WhereFilter filter, FormulaParserConfiguration parserConfiguration) { - Assert.eqTrue(conditionSupported(condition), condition + " is not supported by RangeConditionFilter"); + Assert.eqTrue(conditionSupported(condition), condition + " is not supported by RangeFilter"); this.columnName = columnName; this.condition = condition; this.value = value; @@ -121,7 +121,7 @@ private static Condition conditionFromString(String conditionString) { case ">=": return Condition.GREATER_THAN_OR_EQUAL; default: - throw new IllegalArgumentException(conditionString + " is not supported by RangeConditionFilter"); + throw new IllegalArgumentException(conditionString + " is not supported by RangeFilter"); } } @@ -218,7 +218,7 @@ public void init( if (expression != null) { filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); } else { - throw new IllegalArgumentException("RangeConditionFilter does not support type " + throw new IllegalArgumentException("RangeFilter does not support type " + colClass.getSimpleName() + " for column " + columnName); } } @@ -237,7 +237,7 @@ private static LongRangeFilter makeInstantRangeFilter(String columnName, Conditi case GREATER_THAN_OR_EQUAL: return new InstantRangeFilter(columnName, value, Long.MAX_VALUE, true, true); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } @@ -252,7 +252,7 @@ private static LongRangeFilter makeZonedDateTimeRangeFilter(String columnName, C case GREATER_THAN_OR_EQUAL: return new ZonedDateTimeRangeFilter(columnName, value, Long.MAX_VALUE, true, true); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } @@ -268,7 +268,7 @@ private static SingleSidedComparableRangeFilter makeComparableRangeFilter(String case GREATER_THAN_OR_EQUAL: return new SingleSidedComparableRangeFilter(columnName, comparable, true, true); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 801687aa593..6257a2275b7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -62,7 +62,7 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, case GREATER_THAN_OR_EQUAL: return geq(columnName, value); default: - throw new IllegalArgumentException("RangeConditionFilter does not support condition " + condition); + throw new IllegalArgumentException("RangeFilter does not support condition " + condition); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java index cab22f8e792..8ec35409e0a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterAdapter.java @@ -241,8 +241,8 @@ public PreferredLhsColumnRhsVisitor(ColumnName lhs) { this.lhs = Objects.requireNonNull(lhs); } - // The String vs non-String cases are separated out, as it's necessary in the RangeConditionFilter case to - // wrap String literals with quotes (as that's what RangeConditionFilter expects wrt parsing). MatchFilter + // The String vs non-String cases are separated out, as it's necessary in the RangeFilter case to + // wrap String literals with quotes (as that's what RangeFilter expects wrt parsing). MatchFilter // allows us to pass in the already parsed Object (otherwise, if we were passing strValues we would need to // wrap them) From 9d80d20b4cdd2f2e49c21f182b9153af68256bfa Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 18 Jun 2024 12:41:43 -0600 Subject: [PATCH 23/44] spotlesS --- .../deephaven/engine/table/impl/select/MatchFilter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 97d9533a6c5..f1fa023dd48 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -302,10 +302,10 @@ Object convertParamValue(Object paramValue) { /** * Convert the string value to the appropriate type for the column. * - * @param column the column definition - * @param strValue the string value to convert + * @param column the column definition + * @param strValue the string value to convert * @param queryScopeVariables the query scope variables - * @param valueConsumer the consumer for the converted value + * @param valueConsumer the consumer for the converted value * @return whether the value was an array or collection */ final boolean convertValue( @@ -340,7 +340,7 @@ final boolean convertValue( return false; } } - + public static class ColumnTypeConvertorFactory { public static ColumnTypeConvertor getConvertor(final Class cls) { if (cls == byte.class) { From b58f235ff2a553aef6c2661ff8b0357a93a0d048 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 18 Jun 2024 12:44:35 -0600 Subject: [PATCH 24/44] hard code mirroring --- .../table/impl/select/WhereFilterFactory.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index b029525c687..6a12e683699 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -172,8 +172,22 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a case "<=": case ">=": if (mirrored) { - final String dir = op.substring(0, 1); - op = op.replaceFirst(dir, dir.equals("<") ? ">" : "<"); + switch (op) { + case "<": + op = ">"; + break; + case "<=": + op = ">="; + break; + case ">": + op = "<"; + break; + case ">=": + op = "<="; + break; + default: + throw new IllegalStateException("Unexpected operator: " + op); + } } log.debug().append("WhereFilterFactory creating RangeFilter for expression: ") .append(expression).endl(); From 4fc140f6361a981bad482641d11d10dd88e4e34e Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 18 Jun 2024 16:38:59 -0600 Subject: [PATCH 25/44] Ryan's Comments Continued --- .../java/io/deephaven/time/DateTimeUtils.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java index fe6d549f30f..9bd8529aed7 100644 --- a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java +++ b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java @@ -82,6 +82,13 @@ public class DateTimeUtils { private static final Pattern DATE_TZ_PATTERN = Pattern.compile( "(?[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])(?[tT]?) (?[a-zA-Z_/]+)"); + /** + * Matches dates without time zones. + */ + private static final Pattern LOCAL_DATE_PATTERN = Pattern.compile( + "(?[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])(?[tT]?)"); + + /** * Matches time durations. */ @@ -4377,19 +4384,18 @@ public static Instant parseInstantQuiet(@Nullable final String s) { * Parses the string argument as a {@link LocalDateTime}. *

    * Date time strings are formatted according to the ISO 8601 date time format - * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS] TZ} and others. + * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS]} and others. * * @param s date time string * @return a {@link LocalDateTime} represented by the input string * @throws DateTimeParseException if the string cannot be parsed - * @see DateTimeFormatter#ISO_INSTANT */ @ScriptApi @NotNull public static LocalDateTime parseLocalDateTime(@NotNull final String s) { // noinspection ConstantConditions if (s == null) { - throw new DateTimeParseException("Cannot parse datetime (null): " + s); + throw new DateTimeParseException("Cannot parse local date time (null): " + s); } try { @@ -4399,21 +4405,12 @@ public static LocalDateTime parseLocalDateTime(@NotNull final String s) { } try { - final Matcher dtzMatcher = DATE_TZ_PATTERN.matcher(s); - if (dtzMatcher.matches()) { - final String dateString = dtzMatcher.group("date"); + final Matcher dtMatcher = LOCAL_DATE_PATTERN.matcher(s); + if (dtMatcher.matches()) { + final String dateString = dtMatcher.group("date"); return LocalDate.parse(dateString, FORMATTER_ISO_LOCAL_DATE).atTime(LocalTime.of(0, 0)); } - - final String dateTimeString; - final int spaceIndex = s.indexOf(' '); - if (spaceIndex != -1) { - dateTimeString = s.substring(0, spaceIndex); - } else { - dateTimeString = s; - } - - return LocalDateTime.parse(dateTimeString, FORMATTER_ISO_LOCAL_DATE_TIME); + return LocalDateTime.parse(s, FORMATTER_ISO_LOCAL_DATE_TIME); } catch (Exception ex) { throw new DateTimeParseException("Cannot parse local date time: " + s, ex); } From 5e5d4cb474c95b1cf110a113fe4a66c8b48ebcc3 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 10:21:55 -0600 Subject: [PATCH 26/44] other feedback --- .../engine/table/impl/select/MatchFilter.java | 55 +++++++++++++++++++ .../engine/table/impl/select/RangeFilter.java | 34 +++++++++--- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index f1fa023dd48..34a82c9c94f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -19,6 +19,7 @@ import io.deephaven.engine.table.impl.indexer.DataIndexer; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.QueryConstants; import io.deephaven.util.SafeCloseable; import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.util.type.ArrayTypeUtils; @@ -347,6 +348,9 @@ public static ColumnTypeConvertor getConvertor(final Class cls) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_BYTE".equals(str)) { + return QueryConstants.NULL_BYTE_BOXED; + } return Byte.parseByte(str); } }; @@ -355,6 +359,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_SHORT".equals(str)) { + return QueryConstants.NULL_SHORT_BOXED; + } return Short.parseShort(str); } }; @@ -363,6 +370,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_INT".equals(str)) { + return QueryConstants.NULL_INT_BOXED; + } return Integer.parseInt(str); } }; @@ -371,6 +381,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_LONG".equals(str)) { + return QueryConstants.NULL_LONG_BOXED; + } return Long.parseLong(str); } }; @@ -379,6 +392,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_FLOAT".equals(str)) { + return QueryConstants.NULL_FLOAT_BOXED; + } return Float.parseFloat(str); } }; @@ -387,6 +403,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_DOUBLE".equals(str)) { + return QueryConstants.NULL_DOUBLE_BOXED; + } return Double.parseDouble(str); } }; @@ -396,6 +415,9 @@ Object convertStringLiteral(String str) { @Override Object convertStringLiteral(String str) { // NB: Boolean.parseBoolean(str) doesn't do what we want here - anything not true is false. + if ("null".equals(str) || "NULL_BOOLEAN".equals(str)) { + return QueryConstants.NULL_BOOLEAN; + } if (str.equalsIgnoreCase("true")) { return Boolean.TRUE; } @@ -411,6 +433,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_CHAR".equals(str)) { + return QueryConstants.NULL_CHAR_BOXED; + } if (str.length() > 1) { // TODO: #1517 Allow escaping of chars if (str.length() == 3 && ((str.charAt(0) == '\'' && str.charAt(2) == '\'') @@ -429,6 +454,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } return new BigDecimal(str); } }; @@ -437,6 +465,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } return new BigInteger(str); } }; @@ -508,6 +539,9 @@ Object convertParamValue(Object paramValue) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( "Instant literal not enclosed in single-quotes (\"" + str + "\")"); @@ -520,6 +554,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( "LocalDate literal not enclosed in single-quotes (\"" + str + "\")"); @@ -532,6 +569,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( "LocalTime literal not enclosed in single-quotes (\"" + str + "\")"); @@ -544,6 +584,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( "LocalDateTime literal not enclosed in single-quotes (\"" + str + "\")"); @@ -556,6 +599,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str) || "NULL_BYTE".equals(str)) { + return QueryConstants.NULL_BYTE; + } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( "ZoneDateTime literal not enclosed in single-quotes (\"" + str + "\")"); @@ -568,6 +614,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.startsWith("\"") || str.startsWith("`")) { return str.substring(1, str.length() - 1); } else if (str.contains(".")) { @@ -594,6 +643,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } if (str.startsWith("\"") || str.startsWith("`")) { return DisplayWrapper.make(str.substring(1, str.length() - 1)); } else { @@ -605,6 +657,9 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { + if ("null".equals(str)) { + return null; + } throw new IllegalArgumentException( "Can't create " + cls.getName() + " from String Literal for value auto-conversion"); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index a2731a21266..97836d1220a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -174,24 +174,35 @@ public void init( if (conversionError != null) { if (expression != null) { - filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + try { + filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + } catch (final RuntimeException ignored) { + throw conversionError; + } } else { throw conversionError; } } else if (colClass == double.class || colClass == Double.class) { - filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, (double) realValue.getValue()); + filter = DoubleRangeFilter.makeDoubleRangeFilter(columnName, condition, + TypeUtils.unbox((Double) realValue.getValue())); } else if (colClass == float.class || colClass == Float.class) { - filter = FloatRangeFilter.makeFloatRangeFilter(columnName, condition, (float) realValue.getValue()); + filter = FloatRangeFilter.makeFloatRangeFilter(columnName, condition, + TypeUtils.unbox((Float) realValue.getValue())); } else if (colClass == char.class || colClass == Character.class) { - filter = CharRangeFilter.makeCharRangeFilter(columnName, condition, (char) realValue.getValue()); + filter = CharRangeFilter.makeCharRangeFilter(columnName, condition, + TypeUtils.unbox((Character) realValue.getValue())); } else if (colClass == byte.class || colClass == Byte.class) { - filter = ByteRangeFilter.makeByteRangeFilter(columnName, condition, (byte) realValue.getValue()); + filter = ByteRangeFilter.makeByteRangeFilter(columnName, condition, + TypeUtils.unbox((Byte) realValue.getValue())); } else if (colClass == short.class || colClass == Short.class) { - filter = ShortRangeFilter.makeShortRangeFilter(columnName, condition, (short) realValue.getValue()); + filter = ShortRangeFilter.makeShortRangeFilter(columnName, condition, + TypeUtils.unbox((Short) realValue.getValue())); } else if (colClass == int.class || colClass == Integer.class) { - filter = IntRangeFilter.makeIntRangeFilter(columnName, condition, (int) realValue.getValue()); + filter = IntRangeFilter.makeIntRangeFilter(columnName, condition, + TypeUtils.unbox((Integer) realValue.getValue())); } else if (colClass == long.class || colClass == Long.class) { - filter = LongRangeFilter.makeLongRangeFilter(columnName, condition, (long) realValue.getValue()); + filter = LongRangeFilter.makeLongRangeFilter(columnName, condition, + TypeUtils.unbox((Long) realValue.getValue())); } else if (colClass == Instant.class) { filter = makeInstantRangeFilter(columnName, condition, DateTimeUtils.epochNanos((Instant) realValue.getValue())); @@ -216,7 +227,12 @@ public void init( // The expression looks like a comparison of number, string, or boolean // but the type does not match (or the column type is misconfigured) if (expression != null) { - filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + try { + filter = ConditionFilter.createConditionFilter(expression, parserConfiguration); + } catch (final RuntimeException ignored) { + throw new IllegalArgumentException("RangeFilter does not support type " + + colClass.getSimpleName() + " for column " + columnName); + } } else { throw new IllegalArgumentException("RangeFilter does not support type " + colClass.getSimpleName() + " for column " + columnName); From aff8a28f2087e7c9637d189ec239717ddc5a4461 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 10:58:46 -0600 Subject: [PATCH 27/44] more feedback from rnd1 --- .../engine/table/impl/select/MatchFilter.java | 14 ++++-- .../engine/table/impl/select/RangeFilter.java | 49 ++++++++++++------- .../gui/table/filters/Condition.java | 46 ++++++++++++++--- 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 34a82c9c94f..3b7b51674b0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -56,7 +56,7 @@ static MatchFilter ofLiterals( private final CachingSupplier failoverFilter; @NotNull - private final String columnName; + private String columnName; private Object[] values; private final String[] strValues; private final boolean invertMatch; @@ -187,8 +187,16 @@ public synchronized void init( try { ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { - throw new RuntimeException("Column \"" + columnName - + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); + if (strValues != null && strValues.length == 1 + && (column = tableDefinition.getColumn(strValues[0])) != null) { + // fix up for the case where column name and variable name were swapped + String tmp = columnName; + columnName = strValues[0]; + strValues[0] = tmp; + } else { + throw new RuntimeException("Column \"" + columnName + + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); + } } if (strValues == null) { initialized = true; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 97836d1220a..070d1856817 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -37,9 +37,9 @@ */ public class RangeFilter extends WhereFilterImpl { - private final String columnName; - private final Condition condition; - private final String value; + private String columnName; + private String value; + private Condition condition; // The expression prior to being parsed private final String expression; @@ -148,28 +148,39 @@ public void init( return; } - final ColumnDefinition def = tableDefinition.getColumn(columnName); + RuntimeException conversionError = null; + ColumnDefinition def = tableDefinition.getColumn(columnName); if (def == null) { - throw new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " - + tableDefinition.getColumnNames()); + if ((def = tableDefinition.getColumn(value)) != null) { + // fix up for the case where column name and variable name were swapped + String tmp = columnName; + columnName = value; + value = tmp; + condition = condition.mirror(); + } else{ + conversionError = new RuntimeException("Column \"" + columnName + + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); + } } - final Class colClass = def.getDataType(); + final Class colClass = def == null ? null : def.getDataType(); + final MutableObject realValue = new MutableObject<>(); - final MatchFilter.ColumnTypeConvertor convertor = - MatchFilter.ColumnTypeConvertorFactory.getConvertor(def.getDataType()); + if (def != null) { + final MatchFilter.ColumnTypeConvertor convertor = + MatchFilter.ColumnTypeConvertorFactory.getConvertor(def.getDataType()); - RuntimeException conversionError = null; - final MutableObject realValue = new MutableObject<>(); - try { - boolean wasAnArrayType = convertor.convertValue( - def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); - if (wasAnArrayType) { - throw new IllegalArgumentException("RangeFilter does not support array types for column " - + columnName + " with value <" + value + ">"); + try { + boolean wasAnArrayType = convertor.convertValue( + def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); + if (wasAnArrayType) { + conversionError = + new IllegalArgumentException("RangeFilter does not support array types for column " + + columnName + " with value <" + value + ">"); + } + } catch (final RuntimeException err) { + conversionError = err; } - } catch (final RuntimeException err) { - conversionError = err; } if (conversionError != null) { diff --git a/engine/table/src/main/java/io/deephaven/gui/table/filters/Condition.java b/engine/table/src/main/java/io/deephaven/gui/table/filters/Condition.java index 0c29d40e598..8c7a3c6ab51 100644 --- a/engine/table/src/main/java/io/deephaven/gui/table/filters/Condition.java +++ b/engine/table/src/main/java/io/deephaven/gui/table/filters/Condition.java @@ -20,16 +20,46 @@ public enum Condition { NOT_INCLUDES_MATCH_CASE("not includes (casesen)", false), // Numbers and Dates - LESS_THAN("less than", false), - GREATER_THAN("greater than", false), - LESS_THAN_OR_EQUAL("less than or equal to", false), - GREATER_THAN_OR_EQUAL("greater than or equal to", false), + LESS_THAN("less than", false) { + @Override + public Condition mirror() { + return Condition.GREATER_THAN; + } + }, + GREATER_THAN("greater than", false) { + @Override + public Condition mirror() { + return Condition.LESS_THAN; + } + }, + LESS_THAN_OR_EQUAL("less than or equal to", false) { + @Override + public Condition mirror() { + return Condition.GREATER_THAN_OR_EQUAL; + } + }, + GREATER_THAN_OR_EQUAL("greater than or equal to", false) { + @Override + public Condition mirror() { + return Condition.LESS_THAN_OR_EQUAL; + } + }, // Numbers EQUALS_ABS("equals (abs)", true), NOT_EQUALS_ABS("not equals (abs)", false), - LESS_THAN_ABS("less than (abs)", false), - GREATER_THAN_ABS("greater than (abs)", false), + LESS_THAN_ABS("less than (abs)", false) { + @Override + public Condition mirror() { + return Condition.GREATER_THAN_ABS; + } + }, + GREATER_THAN_ABS("greater than (abs)", false) { + @Override + public Condition mirror() { + return Condition.LESS_THAN_ABS; + } + }, // Lists INCLUDED_IN("included in list", true), @@ -45,4 +75,8 @@ public enum Condition { this.description = description; this.defaultOr = defaultOr; } + + public Condition mirror() { + return this; + } } From cf07196d9784a780fe20548832505b4fadc12937 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 10:59:24 -0600 Subject: [PATCH 28/44] spotless --- .../java/io/deephaven/engine/table/impl/select/RangeFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 070d1856817..10a6f0a7961 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -157,7 +157,7 @@ public void init( columnName = value; value = tmp; condition = condition.mirror(); - } else{ + } else { conversionError = new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " + tableDefinition.getColumnNames()); } From 628fe02392d031f880f2e3210e7f6c39c04e07be Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 11:05:47 -0600 Subject: [PATCH 29/44] quick fix --- .../io/deephaven/engine/table/impl/select/MatchFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 3b7b51674b0..f06b2d23a12 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -607,8 +607,8 @@ Object convertStringLiteral(String str) { return new ColumnTypeConvertor() { @Override Object convertStringLiteral(String str) { - if ("null".equals(str) || "NULL_BYTE".equals(str)) { - return QueryConstants.NULL_BYTE; + if ("null".equals(str)) { + return null; } if (str.charAt(0) != '\'' || str.charAt(str.length() - 1) != '\'') { throw new IllegalArgumentException( From 73ffe40863c74f2f5abce9cf891a9f1ed419375b Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 11:11:28 -0600 Subject: [PATCH 30/44] whitespace fix --- .../engine/table/impl/select/WhereFilterFactoryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java index f5be583ea88..453479692cf 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/WhereFilterFactoryTest.java @@ -62,7 +62,6 @@ public void testInComplex() { } catch (IllegalArgumentException e) { assertEquals(e.getMessage(), "Failed to convert literal value <1> for column \"Maturity\" of type java.lang.String"); - } } From 15d66d5603494667bbdbd1de66ecf89ac59adfc2 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Fri, 21 Jun 2024 13:54:15 -0600 Subject: [PATCH 31/44] Add fallback tests and bugfix --- .../engine/table/impl/select/RangeFilter.java | 4 ++ .../table/impl/QueryTableWhereSingleTest.java | 6 +++ .../table/impl/QueryTableWhereTest.java | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 10a6f0a7961..a17eed54891 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -323,6 +323,10 @@ public void setRecomputeListener(RecomputeListener listener) {} @Override public WhereFilter copy() { + if (filter != null) { + return filter.copy(); + } + return new RangeFilter(columnName, condition, value, expression, filter, parserConfiguration); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java index 7126b72db35..f9ac2c192d3 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java @@ -24,4 +24,10 @@ public void tearDown() throws Exception { QueryTable.FORCE_PARALLEL_WHERE = oldParallel; QueryTable.DISABLE_PARALLEL_WHERE = oldDisable; } + + @Override + public void testRangeFilterFallback() { + super.testRangeFilterFallback(); + } } + diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index 0a8b54965b7..4f9c082e2e7 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -32,13 +32,16 @@ import io.deephaven.engine.testutil.generator.*; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.engine.util.TableTools; +import io.deephaven.gui.table.filters.Condition; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.QueryConstants; import io.deephaven.util.SafeCloseable; import io.deephaven.util.annotations.ReflexiveUse; +import io.deephaven.util.datastructures.CachingSupplier; import junit.framework.TestCase; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableObject; import org.junit.Rule; import org.junit.Test; @@ -1365,4 +1368,43 @@ public void testFilterErrorUpdate() { // The where result should have failed, because the filter expression is invalid for the new data. Assert.eqTrue(whereResult.isFailed(), "whereResult.isFailed()"); } + + @Test + public void testMatchFilterFallback() { + final Table table = emptyTable(10).update("X=i"); + ExecutionContext.getContext().getQueryScope().putParam("var1", 10); + ExecutionContext.getContext().getQueryScope().putParam("var2", 20); + + final MutableBoolean called = new MutableBoolean(false); + final MatchFilter filter = new MatchFilter( + new CachingSupplier<>(() -> { + called.setValue(true); + return ConditionFilter.createConditionFilter("var1 != var2"); + }), + MatchFilter.CaseSensitivity.IgnoreCase, MatchFilter.MatchType.Inverted, "var1", "var2"); + + final Table result = table.where(filter); + assertTableEquals(table, result); + + Assert.eqTrue(called.booleanValue(), "called.booleanValue()"); + + final WhereFilter copyFilter = filter.copy(); + Assert.eqTrue(copyFilter instanceof ConditionFilter, "copyFilter instanceof ConditionFilter"); + } + + @Test + public void testRangeFilterFallback() { + final Table table = emptyTable(10).update("X=i"); + ExecutionContext.getContext().getQueryScope().putParam("var1", 10); + ExecutionContext.getContext().getQueryScope().putParam("var2", 20); + + final RangeFilter filter = new RangeFilter( + "0", Condition.LESS_THAN, "var2", "0 < var2", FormulaParserConfiguration.parser); + + final Table result = table.where(filter); + assertTableEquals(table, result); + + final WhereFilter copyFilter = filter.copy(); + Assert.eqTrue(copyFilter instanceof ConditionFilter, "copyFilter instanceof ConditionFilter"); + } } From 1dbab30517f0b818b1682bb0aca4c741b2a581e2 Mon Sep 17 00:00:00 2001 From: Nate Bauernfeind Date: Mon, 24 Jun 2024 10:35:36 -0600 Subject: [PATCH 32/44] Ryan's rnd2 direct suggestions. Co-authored-by: Ryan Caudy --- .../deephaven/engine/table/impl/select/MatchFilter.java | 9 +-------- .../deephaven/engine/table/impl/select/RangeFilter.java | 4 ---- .../engine/table/impl/QueryTableWhereSingleTest.java | 5 ----- .../src/main/java/io/deephaven/time/DateTimeUtils.java | 3 +-- 4 files changed, 2 insertions(+), 19 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index f06b2d23a12..496c7d2746f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -52,7 +52,7 @@ static MatchFilter ofLiterals( literals.stream().map(AsObject::of).toArray()); } - /** A fail-over where filter supplier should the match filter initialization fail. */ + /** A fail-over WhereFilter supplier should the match filter initialization fail. */ private final CachingSupplier failoverFilter; @NotNull @@ -84,8 +84,6 @@ public enum CaseSensitivity { MatchCase, IgnoreCase } - // TODO NATE NOCOMMIT: USE BUILDER TO CREATE MATCH FILTER?? - public MatchFilter( @NotNull final MatchType matchType, @NotNull final String columnName, @@ -717,11 +715,6 @@ public boolean canMemoize() { public WhereFilter copy() { final MatchFilter copy; if (strValues != null) { - final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; - if (failover != null) { - return failover.copy(); - } - copy = new MatchFilter( failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), columnName, strValues, null); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index a17eed54891..10a6f0a7961 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -323,10 +323,6 @@ public void setRecomputeListener(RecomputeListener listener) {} @Override public WhereFilter copy() { - if (filter != null) { - return filter.copy(); - } - return new RangeFilter(columnName, condition, value, expression, filter, parserConfiguration); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java index f9ac2c192d3..b804f72d313 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java @@ -24,10 +24,5 @@ public void tearDown() throws Exception { QueryTable.FORCE_PARALLEL_WHERE = oldParallel; QueryTable.DISABLE_PARALLEL_WHERE = oldDisable; } - - @Override - public void testRangeFilterFallback() { - super.testRangeFilterFallback(); - } } diff --git a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java index d427c941530..73e4cbbe535 100644 --- a/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java +++ b/engine/time/src/main/java/io/deephaven/time/DateTimeUtils.java @@ -5050,11 +5050,10 @@ public static LocalDateTime parseLocalDateTime(@NotNull final String s) { * Parses the string argument as a {@link LocalDateTime}. *

    * Date time strings are formatted according to the ISO 8601 date time format - * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS] TZ} and others. + * {@code yyyy-MM-ddThh:mm:ss[.SSSSSSSSS]} and others. * * @param s date time string * @return a {@link LocalDateTime} represented by the input string, or {@code null} if the string can not be parsed - * @see DateTimeFormatter#ISO_INSTANT */ @ScriptApi @Nullable From 1befd4da9f4e572124929d5e6f706176350b2d07 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 10:36:55 -0600 Subject: [PATCH 33/44] Ryan's rnd2 feedback --- .../engine/table/impl/DeferredViewTable.java | 4 +- .../engine/table/impl/select/MatchFilter.java | 39 +++++++++++++------ .../engine/table/impl/select/WhereFilter.java | 2 + 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index 37816ccd279..c6aaed349db 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -212,9 +212,7 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } else if (filter instanceof MatchFilter) { final MatchFilter matchFilter = (MatchFilter) filter; Assert.assertion(myRenames.size() == 1, "Match Filters should only use one column!"); - String newName = myRenames.get(matchFilter.getColumnName()); - Assert.neqNull(newName, "newName"); - final MatchFilter newFilter = matchFilter.renameFilter(newName); + final WhereFilter newFilter = matchFilter.renameFilter(myRenames); newFilter.init(tableReference.getDefinition(), compilationProcessor); preViewFilters.add(newFilter); } else if (filter instanceof ConditionFilter) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 496c7d2746f..bdba96504b9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -5,6 +5,7 @@ import io.deephaven.api.literal.Literal; import io.deephaven.base.string.cache.CompressedString; +import io.deephaven.base.verify.Assert; import io.deephaven.engine.liveness.LivenessScopeStack; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; @@ -53,7 +54,7 @@ static MatchFilter ofLiterals( } /** A fail-over WhereFilter supplier should the match filter initialization fail. */ - private final CachingSupplier failoverFilter; + private final CachingSupplier failoverFilter; @NotNull private String columnName; @@ -111,7 +112,7 @@ public MatchFilter( } public MatchFilter( - @Nullable final CachingSupplier failoverFilter, + @Nullable final CachingSupplier failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @@ -120,7 +121,7 @@ public MatchFilter( } private MatchFilter( - @Nullable final CachingSupplier failoverFilter, + @Nullable final CachingSupplier failoverFilter, @NotNull final CaseSensitivity sensitivity, @NotNull final MatchType matchType, @NotNull final String columnName, @@ -134,8 +135,20 @@ private MatchFilter( this.values = values; } - public MatchFilter renameFilter(String newName) { + private ConditionFilter getFailoverFilterIfCached() { + return failoverFilter != null ? failoverFilter.getIfCached() : null; + } + + public WhereFilter renameFilter(Map renames) { + final ConditionFilter failover = getFailoverFilterIfCached(); + if (failover != null) { + return failover.renameFilter(renames); + } + + final String newName = renames.get(columnName); + Assert.neqNull(newName, "newName"); if (strValues == null) { + // when we're constructed with values then there is no failover filter return new MatchFilter(getMatchType(), newName, values); } else { return new MatchFilter( @@ -144,10 +157,6 @@ public MatchFilter renameFilter(String newName) { } } - public String getColumnName() { - return columnName; - } - public Object[] getValues() { return values; } @@ -162,6 +171,13 @@ public MatchType getMatchType() { @Override public List getColumns() { + if (!initialized) { + throw new IllegalStateException("Filter must be initialized to invoke getColumnName"); + } + final WhereFilter failover = getFailoverFilterIfCached(); + if (failover != null) { + return failover.getColumns(); + } return Collections.singletonList(columnName); } @@ -258,7 +274,7 @@ public Stream getDependencyStream() { @Override public WritableRowSet filter( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { - final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + final WhereFilter failover = getFailoverFilterIfCached(); if (failover != null) { return failover.filter(selection, fullSet, table, usePrev); } @@ -271,7 +287,7 @@ public WritableRowSet filter( @Override public WritableRowSet filterInverse( @NotNull RowSet selection, @NotNull RowSet fullSet, @NotNull Table table, boolean usePrev) { - final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + final WhereFilter failover = getFailoverFilterIfCached(); if (failover != null) { return failover.filterInverse(selection, fullSet, table, usePrev); } @@ -282,7 +298,7 @@ public WritableRowSet filterInverse( @Override public boolean isSimpleFilter() { - final WhereFilter failover = failoverFilter != null ? failoverFilter.getIfCached() : null; + final WhereFilter failover = getFailoverFilterIfCached(); if (failover != null) { return failover.isSimpleFilter(); } @@ -719,6 +735,7 @@ public WhereFilter copy() { failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), columnName, strValues, null); } else { + // when we're constructed with values then there is no failover filter copy = new MatchFilter(getMatchType(), columnName, values); } if (initialized) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index dbe6b627903..e4510531018 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -87,6 +87,8 @@ interface RecomputeListener { /** * Get the columns required by this select filter. + *

    + * This filter must already be initialized before calling this method. * * @return the columns used as input by this select filter. */ From 1ddde7d99d611d6ee507dc265f7c0c58f66fbb3d Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 10:44:55 -0600 Subject: [PATCH 34/44] Quick compilation fixes --- .../engine/table/impl/select/WhereFilterFactory.java | 4 ++-- .../io/deephaven/engine/table/impl/QueryTableWhereTest.java | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 6a12e683699..148e210304c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -160,8 +160,8 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( - new CachingSupplier<>( - () -> ConditionFilter.createConditionFilter(expression, parserConfiguration)), + new CachingSupplier<>(() -> (ConditionFilter) + ConditionFilter.createConditionFilter(expression, parserConfiguration)), CaseSensitivity.MatchCase, inverted ? MatchType.Inverted : MatchType.Regular, columnName, diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index 4f9c082e2e7..108467f3f1c 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -1379,7 +1379,7 @@ public void testMatchFilterFallback() { final MatchFilter filter = new MatchFilter( new CachingSupplier<>(() -> { called.setValue(true); - return ConditionFilter.createConditionFilter("var1 != var2"); + return (ConditionFilter) ConditionFilter.createConditionFilter("var1 != var2"); }), MatchFilter.CaseSensitivity.IgnoreCase, MatchFilter.MatchType.Inverted, "var1", "var2"); @@ -1387,9 +1387,6 @@ public void testMatchFilterFallback() { assertTableEquals(table, result); Assert.eqTrue(called.booleanValue(), "called.booleanValue()"); - - final WhereFilter copyFilter = filter.copy(); - Assert.eqTrue(copyFilter instanceof ConditionFilter, "copyFilter instanceof ConditionFilter"); } @Test From 24315c0b5ee746d9fab57696b32f4e9a10f61500 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 11:20:40 -0600 Subject: [PATCH 35/44] Few More Filter Delegation Fixes --- .../engine/table/impl/select/MatchFilter.java | 10 +++++++++- .../engine/table/impl/select/RangeFilter.java | 17 ++++++++++++++--- .../engine/table/impl/select/WhereFilter.java | 2 ++ .../engine/table/impl/QueryTableWhereTest.java | 4 ++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index bdba96504b9..e2f5705ecc1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -183,6 +183,13 @@ public List getColumns() { @Override public List getColumnArrays() { + if (!initialized) { + throw new IllegalStateException("Filter must be initialized to invoke getColumnArrays"); + } + final WhereFilter failover = getFailoverFilterIfCached(); + if (failover != null) { + return failover.getColumnArrays(); + } return Collections.emptyList(); } @@ -732,7 +739,8 @@ public WhereFilter copy() { final MatchFilter copy; if (strValues != null) { copy = new MatchFilter( - failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, + failoverFilter == null ? null : new CachingSupplier<>(() -> failoverFilter.get().copy()), + caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), columnName, strValues, null); } else { // when we're constructed with values then there is no failover filter diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 10a6f0a7961..9cd416a20a3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.RowSet; import io.deephaven.gui.table.filters.Condition; +import io.deephaven.util.annotations.VisibleForTesting; import io.deephaven.util.type.TypeUtils; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; @@ -23,7 +24,6 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZonedDateTime; -import java.util.Collections; import java.util.List; /** @@ -127,12 +127,23 @@ private static Condition conditionFromString(String conditionString) { @Override public List getColumns() { - return Collections.singletonList(columnName); + if (filter == null) { + throw new IllegalStateException("Filter must be initialized to invoke getColumnName"); + } + return filter.getColumns(); } @Override public List getColumnArrays() { - return Collections.emptyList(); + if (filter == null) { + throw new IllegalStateException("Filter must be initialized to invoke getColumnArrays"); + } + return filter.getColumnArrays(); + } + + @VisibleForTesting + public WhereFilter getRealFilter() { + return filter; } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index e4510531018..5c17955ccde 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -96,6 +96,8 @@ interface RecomputeListener { /** * Get the array columns required by this select filter. + *

    + * This filter must already be initialized before calling this method. * * @return the columns used as array input by this select filter. */ diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index 108467f3f1c..e800401c839 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -1401,7 +1401,7 @@ public void testRangeFilterFallback() { final Table result = table.where(filter); assertTableEquals(table, result); - final WhereFilter copyFilter = filter.copy(); - Assert.eqTrue(copyFilter instanceof ConditionFilter, "copyFilter instanceof ConditionFilter"); + final WhereFilter realFilter = filter.getRealFilter(); + Assert.eqTrue(realFilter instanceof ConditionFilter, "realFilter instanceof ConditionFilter"); } } From dd801eb76f8aac501834789dce587ae3b32506c9 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 11:37:01 -0600 Subject: [PATCH 36/44] DateTimeUtil Tests --- .../table/impl/QueryTableWhereSingleTest.java | 1 - .../io/deephaven/time/TestDateTimeUtils.java | 132 ++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java index b804f72d313..7126b72db35 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereSingleTest.java @@ -25,4 +25,3 @@ public void tearDown() throws Exception { QueryTable.DISABLE_PARALLEL_WHERE = oldDisable; } } - diff --git a/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java b/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java index 8515f0f94bb..7dcad0ad771 100644 --- a/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java +++ b/engine/time/src/test/java/io/deephaven/time/TestDateTimeUtils.java @@ -566,6 +566,138 @@ public void testParseInstantQuiet() { TestCase.assertEquals(dt1s, DateTimeUtils.parseInstantQuiet(Long.toString(seconds))); } + public void testParseLocalDateTime() { + final String[] roots = { + "2010-01-01T12:11", + "2010-01-01T12:00:02", + "2010-01-01T12:00:00.1", + "2010-01-01T12:00:00.123", + "2010-01-01T12:00:00.123", + "2010-01-01T12:00:00.123456789", + }; + + for (String root : roots) { + final LocalDateTime ldt = LocalDateTime.parse(root); + TestCase.assertEquals("LocalDateTime string: " + root, ldt, DateTimeUtils.parseLocalDateTime(root)); + } + + final String[] uglyRoots = { + "2023-04-30", + "2023-04-30T", + "2023-04-30t", + "2023-04-30T9:30:00", + "2023-4-3T9:3:6", + "2023-4-3T9:3", + "2023-4-3T9:3:6.1", + "2023-4-3T9:3:6.123", + "2023-4-3T9:3:6.123456789", + }; + + final LocalDateTime[] uglyLDTs = { + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 9, 30, 0), + LocalDateTime.of(2023, 4, 3, 9, 3, 6), + LocalDateTime.of(2023, 4, 3, 9, 3, 0), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 100_000_000), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 123_000_000), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 123456789), + }; + + for (int i = 0; i < uglyRoots.length; i++) { + final String root = uglyRoots[i]; + final LocalDateTime ldt = uglyLDTs[i]; + TestCase.assertEquals("LocalDateTime string: " + root, ldt, DateTimeUtils.parseLocalDateTime(root)); + } + + try { + DateTimeUtils.parseLocalDateTime("JUNK"); + TestCase.fail("Should throw an exception"); + } catch (Exception ex) { + // pass + } + + try { + DateTimeUtils.parseLocalDateTime("2010-01-01JUNK12:11"); + TestCase.fail("Should throw an exception"); + } catch (Exception ex) { + // pass + } + + try { + DateTimeUtils.parseLocalDateTime("2010-01-01T12:11 JUNK"); + TestCase.fail("Should throw an exception"); + } catch (Exception ex) { + // pass + } + + try { + // noinspection ConstantConditions + DateTimeUtils.parseLocalDateTime(null); + TestCase.fail("Should throw an exception"); + } catch (Exception ex) { + // pass + } + + final String iso8601 = "2022-04-26T00:30:31.087360"; + assertEquals(LocalDateTime.parse(iso8601), DateTimeUtils.parseLocalDateTime(iso8601)); + } + + public void testParseLocalDateTimeQuiet() { + final String[] roots = { + "2010-01-01T12:11", + "2010-01-01T12:00:02", + "2010-01-01T12:00:00.1", + "2010-01-01T12:00:00.123", + "2010-01-01T12:00:00.123", + "2010-01-01T12:00:00.123456789", + }; + + for (String root : roots) { + final LocalDateTime ldt = LocalDateTime.parse(root); + TestCase.assertEquals("LocalDateTime string: " + root, ldt, DateTimeUtils.parseLocalDateTime(root)); + } + + final String[] uglyRoots = { + "2023-04-30", + "2023-04-30T", + "2023-04-30t", + "2023-04-30T9:30:00", + "2023-4-3T9:3:6", + "2023-4-3T9:3", + "2023-4-3T9:3:6.1", + "2023-4-3T9:3:6.123", + "2023-4-3T9:3:6.123456789", + }; + + final LocalDateTime[] uglyLDTs = { + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 0, 0), + LocalDateTime.of(2023, 4, 30, 9, 30, 0), + LocalDateTime.of(2023, 4, 3, 9, 3, 6), + LocalDateTime.of(2023, 4, 3, 9, 3, 0), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 100_000_000), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 123_000_000), + LocalDateTime.of(2023, 4, 3, 9, 3, 6, 123456789), + }; + + for (int i = 0; i < uglyRoots.length; i++) { + final String root = uglyRoots[i]; + final LocalDateTime ldt = uglyLDTs[i]; + TestCase.assertEquals("LocalDateTime string: " + root, ldt, DateTimeUtils.parseLocalDateTime(root)); + } + + TestCase.assertNull(DateTimeUtils.parseLocalDateTimeQuiet("JUNK")); + TestCase.assertNull(DateTimeUtils.parseLocalDateTimeQuiet("2010-01-01JUNK12:11")); + TestCase.assertNull(DateTimeUtils.parseLocalDateTimeQuiet("2010-01-01T12:11 JUNK")); + TestCase.assertNull(DateTimeUtils.parseLocalDateTimeQuiet(null)); + + final String iso8601 = "2022-04-26T00:30:31.087360"; + assertEquals(LocalDateTime.parse(iso8601), DateTimeUtils.parseLocalDateTime(iso8601)); + } + public void testParseZonedDateTime() { final String[] tzs = { "NY", From eaf10ee11faaa51d00340e30f532d439a307d160 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 11:44:04 -0600 Subject: [PATCH 37/44] spotless --- .../engine/table/impl/select/WhereFilterFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 148e210304c..11ff3114cdf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -160,8 +160,8 @@ public WhereFilter getExpression(String expression, Matcher matcher, Object... a log.debug().append("WhereFilterFactory creating MatchFilter for expression: ").append(expression) .endl(); return new MatchFilter( - new CachingSupplier<>(() -> (ConditionFilter) - ConditionFilter.createConditionFilter(expression, parserConfiguration)), + new CachingSupplier<>(() -> (ConditionFilter) ConditionFilter.createConditionFilter(expression, + parserConfiguration)), CaseSensitivity.MatchCase, inverted ? MatchType.Inverted : MatchType.Regular, columnName, From e92cc09146cd70a9c01e0a80876c503205381c06 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 12:22:35 -0600 Subject: [PATCH 38/44] Ryan's rnd3 feedback --- .../deephaven/engine/table/impl/select/MatchFilter.java | 9 +++------ .../deephaven/engine/table/impl/select/RangeFilter.java | 3 ++- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index e2f5705ecc1..3114cdcf664 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -140,11 +140,6 @@ private ConditionFilter getFailoverFilterIfCached() { } public WhereFilter renameFilter(Map renames) { - final ConditionFilter failover = getFailoverFilterIfCached(); - if (failover != null) { - return failover.renameFilter(renames); - } - final String newName = renames.get(columnName); Assert.neqNull(newName, "newName"); if (strValues == null) { @@ -152,7 +147,9 @@ public WhereFilter renameFilter(Map renames) { return new MatchFilter(getMatchType(), newName, values); } else { return new MatchFilter( - failoverFilter, caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, + failoverFilter != null ? new CachingSupplier<>( + () -> failoverFilter.get().renameFilter(renames)) : null, + caseInsensitive ? CaseSensitivity.IgnoreCase : CaseSensitivity.MatchCase, getMatchType(), newName, strValues, null); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 9cd416a20a3..b09424506be 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -334,7 +334,8 @@ public void setRecomputeListener(RecomputeListener listener) {} @Override public WhereFilter copy() { - return new RangeFilter(columnName, condition, value, expression, filter, parserConfiguration); + final WhereFilter innerCopy = filter == null ? null : filter.copy(); + return new RangeFilter(columnName, condition, value, expression, innerCopy, parserConfiguration); } @Override From 3d211fc40eae324c71e1b84ee2aa9ff116833435 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 12:42:52 -0600 Subject: [PATCH 39/44] Make ZonedDateTimeRangeFilter compare ZoneDateTimes --- .../engine/table/impl/select/RangeFilter.java | 18 +-- .../impl/select/ZonedDateTimeRangeFilter.java | 109 ------------------ 2 files changed, 1 insertion(+), 126 deletions(-) delete mode 100644 engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index b09424506be..1b20fef7222 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -235,8 +235,7 @@ public void init( } else if (colClass == LocalDateTime.class) { filter = makeComparableRangeFilter(columnName, condition, (LocalDateTime) realValue.getValue()); } else if (colClass == ZonedDateTime.class) { - filter = makeZonedDateTimeRangeFilter(columnName, condition, - DateTimeUtils.epochNanos((ZonedDateTime) realValue.getValue())); + filter = makeComparableRangeFilter(columnName, condition, (ZonedDateTime) realValue.getValue()); } else if (BigDecimal.class.isAssignableFrom(colClass)) { filter = makeComparableRangeFilter(columnName, condition, (BigDecimal) realValue.getValue()); } else if (BigInteger.class.isAssignableFrom(colClass)) { @@ -279,21 +278,6 @@ private static LongRangeFilter makeInstantRangeFilter(String columnName, Conditi } } - private static LongRangeFilter makeZonedDateTimeRangeFilter(String columnName, Condition condition, long value) { - switch (condition) { - case LESS_THAN: - return new ZonedDateTimeRangeFilter(columnName, value, Long.MIN_VALUE, true, false); - case LESS_THAN_OR_EQUAL: - return new ZonedDateTimeRangeFilter(columnName, value, Long.MIN_VALUE, true, true); - case GREATER_THAN: - return new ZonedDateTimeRangeFilter(columnName, value, Long.MAX_VALUE, false, true); - case GREATER_THAN_OR_EQUAL: - return new ZonedDateTimeRangeFilter(columnName, value, Long.MAX_VALUE, true, true); - default: - throw new IllegalArgumentException("RangeFilter does not support condition " + condition); - } - } - private static SingleSidedComparableRangeFilter makeComparableRangeFilter(String columnName, Condition condition, Comparable comparable) { switch (condition) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java deleted file mode 100644 index ac44e7b8504..00000000000 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ZonedDateTimeRangeFilter.java +++ /dev/null @@ -1,109 +0,0 @@ -// -// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending -// -package io.deephaven.engine.table.impl.select; - -import io.deephaven.base.verify.Assert; -import io.deephaven.engine.table.ColumnDefinition; -import io.deephaven.engine.table.TableDefinition; -import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; -import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; -import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.impl.sources.ReinterpretUtils; -import io.deephaven.chunk.*; -import io.deephaven.chunk.attributes.Values; -import io.deephaven.engine.rowset.WritableRowSet; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.time.DateTimeUtils; -import org.jetbrains.annotations.NotNull; - -import java.time.ZonedDateTime; - -public class ZonedDateTimeRangeFilter extends LongRangeFilter { - - public ZonedDateTimeRangeFilter(String columnName, ZonedDateTime val1, ZonedDateTime val2) { - super(columnName, DateTimeUtils.epochNanos(val1), DateTimeUtils.epochNanos(val2), true, true); - } - - public ZonedDateTimeRangeFilter( - String columnName, ZonedDateTime val1, ZonedDateTime val2, boolean lowerInclusive, boolean upperInclusive) { - super(columnName, DateTimeUtils.epochNanos(val1), DateTimeUtils.epochNanos(val2), - lowerInclusive, upperInclusive); - } - - public ZonedDateTimeRangeFilter( - String columnName, long val1, long val2, boolean lowerInclusive, boolean upperInclusive) { - super(columnName, val1, val2, lowerInclusive, upperInclusive); - } - - @Override - public void init(@NotNull final TableDefinition tableDefinition) { - if (chunkFilter != null) { - return; - } - - final ColumnDefinition def = tableDefinition.getColumn(columnName); - if (def == null) { - throw new RuntimeException("Column \"" + columnName + "\" doesn't exist in this table, available columns: " - + tableDefinition.getColumnNames()); - } - - final Class colClass = def.getDataType(); - Assert.eq(colClass, "colClass", ZonedDateTime.class); - - longFilter = super.initChunkFilter(); - chunkFilter = new ZonedDateTimeLongChunkFilterAdapter(); - } - - @Override - public ZonedDateTimeRangeFilter copy() { - final ZonedDateTimeRangeFilter copy = - new ZonedDateTimeRangeFilter(columnName, lower, upper, lowerInclusive, upperInclusive); - copy.chunkFilter = chunkFilter; - copy.longFilter = longFilter; - return copy; - } - - @Override - public String toString() { - return "ZonedDateTimeRangeFilter(" + columnName + " in " - + (lowerInclusive ? "[" : "(") - + DateTimeUtils.epochNanosToInstant(lower) + "," + DateTimeUtils.epochNanosToInstant(upper) - + (upperInclusive ? "]" : ")") + ")"; - } - - @NotNull - @Override - WritableRowSet binarySearch( - @NotNull final RowSet selection, - @NotNull final ColumnSource columnSource, - final boolean usePrev, - final boolean reverse) { - if (selection.isEmpty()) { - return selection.copy(); - } - - // noinspection unchecked - final ColumnSource zdtColumnSource = - ReinterpretUtils.zonedDateTimeToLongSource((ColumnSource) columnSource); - return super.binarySearch(selection, zdtColumnSource, usePrev, reverse); - } - - private class ZonedDateTimeLongChunkFilterAdapter implements ChunkFilter { - @Override - public void filter(Chunk values, LongChunk keys, - WritableLongChunk results) { - try (final WritableLongChunk writableLongChunk = - WritableLongChunk.makeWritableChunk(values.size())) { - - final ObjectChunk objectValues = values.asObjectChunk(); - for (int ii = 0; ii < values.size(); ++ii) { - final ZonedDateTime zdt = objectValues.get(ii); - writableLongChunk.set(ii, DateTimeUtils.epochNanos(zdt)); - } - writableLongChunk.setSize(values.size()); - longFilter.filter(writableLongChunk, keys, results); - } - } - } -} From c87ba9a646c7627564f5ad992562b97c6e64a11d Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 14:58:10 -0600 Subject: [PATCH 40/44] Force column names to take precedence --- .../engine/table/impl/select/MatchFilter.java | 17 ++++++++++++++--- .../engine/table/impl/select/RangeFilter.java | 3 ++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 3114cdcf664..2568b9d7d10 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -224,7 +224,7 @@ public synchronized void init( final Map queryScopeVariables = compilationProcessor.getQueryScopeVariables(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType()); for (String strValue : strValues) { - convertor.convertValue(column, strValue, queryScopeVariables, valueList::add); + convertor.convertValue(column, tableDefinition, strValue, queryScopeVariables, valueList::add); } values = valueList.toArray(); } catch (final RuntimeException err) { @@ -337,9 +337,17 @@ Object convertParamValue(Object paramValue) { */ final boolean convertValue( @NotNull final ColumnDefinition column, + @NotNull final TableDefinition tableDefinition, @NotNull final String strValue, @NotNull final Map queryScopeVariables, @NotNull final Consumer valueConsumer) { + if (tableDefinition.getColumn(strValue) != null) { + // this is also a column name which needs to take precedence, and we can't convert it + throw new IllegalArgumentException(String.format( + "Failed to convert literal value <%s> for column \"%s\" of type %s; it is a column name", + strValue, column.getName(), column.getDataType().getName())); + } + if (queryScopeVariables.containsKey(strValue)) { Object paramValue = queryScopeVariables.get(strValue); if (paramValue != null && paramValue.getClass().isArray()) { @@ -358,12 +366,15 @@ final boolean convertValue( valueConsumer.accept(convertParamValue(paramValue)); return false; } + try { valueConsumer.accept(convertStringLiteral(strValue)); } catch (Throwable t) { - throw new IllegalArgumentException("Failed to convert literal value <" + strValue + - "> for column \"" + column.getName() + "\" of type " + column.getDataType().getName(), t); + throw new IllegalArgumentException(String.format( + "Failed to convert literal value <%s> for column \"%s\" of type %s", + strValue, column.getName(), column.getDataType().getName()), t); } + return false; } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java index 1b20fef7222..65d65b75e03 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeFilter.java @@ -183,7 +183,8 @@ public void init( try { boolean wasAnArrayType = convertor.convertValue( - def, value, compilationProcessor.getQueryScopeVariables(), realValue::setValue); + def, tableDefinition, value, compilationProcessor.getQueryScopeVariables(), + realValue::setValue); if (wasAnArrayType) { conversionError = new IllegalArgumentException("RangeFilter does not support array types for column " From e07f2f0f4fb430b0cb7b1d689972bb1a81844734 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Mon, 24 Jun 2024 16:58:24 -0600 Subject: [PATCH 41/44] Fix type coercion from one numeric to another --- .../engine/table/impl/select/MatchFilter.java | 95 ++++++++++- .../table/impl/QueryTableWhereTest.java | 147 ++++++++++++++++++ 2 files changed, 241 insertions(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 2568b9d7d10..7d0f17a297b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -15,6 +15,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.engine.table.impl.indexer.DataIndexer; @@ -24,6 +25,7 @@ import io.deephaven.util.SafeCloseable; import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.util.type.ArrayTypeUtils; +import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.jpy.PyObject; @@ -344,7 +346,14 @@ final boolean convertValue( if (tableDefinition.getColumn(strValue) != null) { // this is also a column name which needs to take precedence, and we can't convert it throw new IllegalArgumentException(String.format( - "Failed to convert literal value <%s> for column \"%s\" of type %s; it is a column name", + "Failed to convert value <%s> for column \"%s\" of type %s; it is a column name", + strValue, column.getName(), column.getDataType().getName())); + } + if (strValue.endsWith("_") + && tableDefinition.getColumn(strValue.substring(0, strValue.length() - 1)) != null) { + // this also a column array name which needs to take precedence, and we can't convert it + throw new IllegalArgumentException(String.format( + "Failed to convert value <%s> for column \"%s\" of type %s; it is a column array access name", strValue, column.getName(), column.getDataType().getName())); } @@ -390,6 +399,18 @@ Object convertStringLiteral(String str) { } return Byte.parseByte(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Byte) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.byteCast(boxer.get(paramValue)); + } }; } if (cls == short.class) { @@ -401,6 +422,18 @@ Object convertStringLiteral(String str) { } return Short.parseShort(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Short) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.shortCast(boxer.get(paramValue)); + } }; } if (cls == int.class) { @@ -412,6 +445,18 @@ Object convertStringLiteral(String str) { } return Integer.parseInt(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Integer) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.intCast(boxer.get(paramValue)); + } }; } if (cls == long.class) { @@ -423,6 +468,18 @@ Object convertStringLiteral(String str) { } return Long.parseLong(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Long) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.longCast(boxer.get(paramValue)); + } }; } if (cls == float.class) { @@ -434,6 +491,18 @@ Object convertStringLiteral(String str) { } return Float.parseFloat(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Float) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.floatCast(boxer.get(paramValue)); + } }; } if (cls == double.class) { @@ -445,6 +514,18 @@ Object convertStringLiteral(String str) { } return Double.parseDouble(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Double) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.doubleCast(boxer.get(paramValue)); + } }; } if (cls == Boolean.class) { @@ -485,6 +566,18 @@ Object convertStringLiteral(String str) { } return str.charAt(0); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof Character) { + return paramValue; + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + return QueryLanguageFunctionUtils.charCast(boxer.get(paramValue)); + } }; } if (cls == BigDecimal.class) { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index e800401c839..599df3684b2 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -43,6 +43,7 @@ import junit.framework.TestCase; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableObject; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -1404,4 +1405,150 @@ public void testRangeFilterFallback() { final WhereFilter realFilter = filter.getRealFilter(); Assert.eqTrue(realFilter instanceof ConditionFilter, "realFilter instanceof ConditionFilter"); } + + @Test + public void testEnsureColumnsTakePrecedence() { + final Table table = emptyTable(10).update("X=i", "Y=i%2"); + ExecutionContext.getContext().getQueryScope().putParam("Y", 5); + + { + final Table r1 = table.where("X == Y"); + final Table r2 = table.where("Y == X"); + Assert.equals(r1.getRowSet(), "r1.getRowSet()", RowSetFactory.flat(2)); + assertTableEquals(r1, r2); + } + + { + final Table r1 = table.where("X >= Y"); + final Table r2 = table.where("Y <= X"); + Assert.equals(r1.getRowSet(), "r1.getRowSet()", RowSetFactory.flat(10)); + assertTableEquals(r1, r2); + } + + { + final Table r1 = table.where("X > Y"); + final Table r2 = table.where("Y < X"); + Assert.equals(r1.getRowSet(), "r1.getRowSet()", RowSetFactory.fromRange(2, 9)); + assertTableEquals(r1, r2); + } + + { + final Table r1 = table.where("X < Y"); + final Table r2 = table.where("Y > X"); + Assert.equals(r1.getRowSet(), "r1.getRowSet()", RowSetFactory.empty()); + assertTableEquals(r1, r2); + } + + { + final Table r1 = table.where("X <= Y"); + final Table r2 = table.where("Y >= X"); + Assert.equals(r1.getRowSet(), "r1.getRowSet()", RowSetFactory.flat(2)); + assertTableEquals(r1, r2); + } + } + + @Test + @Ignore + public void testEnsureColumnArraysTakePrecedence() { + // TODO: column arrays aren't well supported in match arrays and this example's where filter fails to compile + final Table table = emptyTable(10).update("X=i", "Y=new int[]{1, 5, 9}"); + ExecutionContext.getContext().getQueryScope().putParam("Y_", new int[] {0, 4, 8}); + + final Table result = table.where("X == Y_[1]"); + Assert.equals(result.getRowSet(), "result.getRowSet()", RowSetFactory.fromKeys(5)); + + // check that the mirror matches the expected result + final Table mResult = table.where("Y_[1] == X"); + assertTableEquals(result, mResult); + } + + @Test + public void testIntToByteCoercion() { + final Table table = emptyTable(11).update("X = ii % 2 == 0 ? (byte) ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", byte.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } + + @Test + public void testIntToShortCoercion() { + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? (short) ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", short.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } + + @Test + public void testLongToIntCoercion() { + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? (int) ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", int.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_LONG); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5L); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } + + @Test + public void testIntToLongCoercion() { + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", long.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } + + @Test + public void testIntToFloatCoercion() { + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? (float) ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", float.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } + + @Test + public void testIntToDoubleCoercion() { + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? (double) ii : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", double.class); + + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table null_result = table.where("X == val_null"); + final Table range_result = table.where("X >= val_5"); + Assert.eq(null_result.size(), "null_result.size()", 5); + Assert.eq(range_result.size(), "range_result.size()", 3); + } } From f18e93e749ae2d354221712f4078f76f35790974 Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 25 Jun 2024 08:48:16 -0600 Subject: [PATCH 42/44] Make actual null work properly w/coercion --- .../engine/table/impl/select/MatchFilter.java | 14 ++++---- .../table/impl/QueryTableWhereTest.java | 36 +++++++++++++++---- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 7d0f17a297b..da5ce2ddca5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -403,7 +403,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Byte) { + if (paramValue instanceof Byte || paramValue == null) { return paramValue; } // noinspection unchecked @@ -426,7 +426,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Short) { + if (paramValue instanceof Short || paramValue == null) { return paramValue; } // noinspection unchecked @@ -449,7 +449,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Integer) { + if (paramValue instanceof Integer || paramValue == null) { return paramValue; } // noinspection unchecked @@ -472,7 +472,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Long) { + if (paramValue instanceof Long || paramValue == null) { return paramValue; } // noinspection unchecked @@ -495,7 +495,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Float) { + if (paramValue instanceof Float || paramValue == null) { return paramValue; } // noinspection unchecked @@ -518,7 +518,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Double) { + if (paramValue instanceof Double || paramValue == null) { return paramValue; } // noinspection unchecked @@ -570,7 +570,7 @@ Object convertStringLiteral(String str) { @Override Object convertParamValue(Object paramValue) { paramValue = super.convertParamValue(paramValue); - if (paramValue instanceof Character) { + if (paramValue instanceof Character || paramValue == null) { return paramValue; } // noinspection unchecked diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index 599df3684b2..fbff142cd3f 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -1468,12 +1468,16 @@ public void testIntToByteCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", byte.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } @@ -1483,12 +1487,16 @@ public void testIntToShortCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", short.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } @@ -1498,12 +1506,16 @@ public void testLongToIntCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", int.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_LONG); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5L); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } @@ -1513,12 +1525,16 @@ public void testIntToLongCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", long.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } @@ -1528,12 +1544,16 @@ public void testIntToFloatCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", float.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } @@ -1543,12 +1563,16 @@ public void testIntToDoubleCoercion() { final Class colType = table.getDefinition().getColumn("X").getDataType(); Assert.eq(colType, "colType", double.class); + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + final Table real_null_result = table.where("X == real_null"); final Table null_result = table.where("X == val_null"); - final Table range_result = table.where("X >= val_5"); Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } } From 45c4c357cac6924230504ad36d743e476e02f63d Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 25 Jun 2024 09:28:40 -0600 Subject: [PATCH 43/44] Also coerce BigDecimal and BigInteger --- .../engine/table/impl/select/MatchFilter.java | 44 ++++++++ .../table/impl/QueryTableWhereTest.java | 102 +++++++++++++----- 2 files changed, 121 insertions(+), 25 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index da5ce2ddca5..d4026f087aa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -589,6 +589,28 @@ Object convertStringLiteral(String str) { } return new BigDecimal(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof BigDecimal || paramValue == null) { + return paramValue; + } + if (paramValue instanceof BigInteger) { + return new BigDecimal((BigInteger) paramValue); + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + final Object boxedValue = boxer.get(paramValue); + if (boxedValue == null) { + return null; + } + if (boxedValue instanceof Number) { + return BigDecimal.valueOf(((Number) boxedValue).doubleValue()); + } + return paramValue; + } }; } if (cls == BigInteger.class) { @@ -600,6 +622,28 @@ Object convertStringLiteral(String str) { } return new BigInteger(str); } + + @Override + Object convertParamValue(Object paramValue) { + paramValue = super.convertParamValue(paramValue); + if (paramValue instanceof BigInteger || paramValue == null) { + return paramValue; + } + if (paramValue instanceof BigDecimal) { + return ((BigDecimal) paramValue).toBigInteger(); + } + // noinspection unchecked + final TypeUtils.TypeBoxer boxer = + (TypeUtils.TypeBoxer) TypeUtils.getTypeBoxer(paramValue.getClass()); + final Object boxedValue = boxer.get(paramValue); + if (boxedValue == null) { + return null; + } + if (boxedValue instanceof Number) { + return BigInteger.valueOf(((Number) boxedValue).longValue()); + } + return paramValue; + } }; } if (cls == String.class) { diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index fbff142cd3f..145141b2b32 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -387,31 +387,31 @@ public void testWhereDynamicIn() { }); } - @Test - public void testWhereDynamicInIncremental() { - testWhereDynamicIncrementalInternal(false, false, true, true); - testWhereDynamicIncrementalInternal(false, false, true, false); - testWhereDynamicIncrementalInternal(false, false, false, true); - testWhereDynamicIncrementalInternal(false, false, false, false); - } - - @Test - public void testWhereDynamicInIncrementalIndexed() { - testWhereDynamicIncrementalInternal(true, false, true, true); - testWhereDynamicIncrementalInternal(true, false, true, false); - testWhereDynamicIncrementalInternal(true, false, false, true); - testWhereDynamicIncrementalInternal(true, false, false, false); - - testWhereDynamicIncrementalInternal(false, true, true, true); - testWhereDynamicIncrementalInternal(false, true, true, false); - testWhereDynamicIncrementalInternal(false, true, false, true); - testWhereDynamicIncrementalInternal(false, true, false, false); - - testWhereDynamicIncrementalInternal(true, true, true, true); - testWhereDynamicIncrementalInternal(true, true, true, false); - testWhereDynamicIncrementalInternal(true, true, false, true); - testWhereDynamicIncrementalInternal(true, true, false, false); - } + // @Test + // public void testWhereDynamicInIncremental() { + // testWhereDynamicIncrementalInternal(false, false, true, true); + // testWhereDynamicIncrementalInternal(false, false, true, false); + // testWhereDynamicIncrementalInternal(false, false, false, true); + // testWhereDynamicIncrementalInternal(false, false, false, false); + // } + // + // @Test + // public void testWhereDynamicInIncrementalIndexed() { + // testWhereDynamicIncrementalInternal(true, false, true, true); + // testWhereDynamicIncrementalInternal(true, false, true, false); + // testWhereDynamicIncrementalInternal(true, false, false, true); + // testWhereDynamicIncrementalInternal(true, false, false, false); + // + // testWhereDynamicIncrementalInternal(false, true, true, true); + // testWhereDynamicIncrementalInternal(false, true, true, false); + // testWhereDynamicIncrementalInternal(false, true, false, true); + // testWhereDynamicIncrementalInternal(false, true, false, false); + // + // testWhereDynamicIncrementalInternal(true, true, true, true); + // testWhereDynamicIncrementalInternal(true, true, true, false); + // testWhereDynamicIncrementalInternal(true, true, false, true); + // testWhereDynamicIncrementalInternal(true, true, false, false); + // } private static void testWhereDynamicIncrementalInternal( boolean filterIndexed, @@ -1575,4 +1575,56 @@ public void testIntToDoubleCoercion() { final Table range_result = table.where("X >= val_5"); Assert.eq(range_result.size(), "range_result.size()", 3); } + + @Test + public void testBigIntegerCoercion() { + ExecutionContext.getContext().getQueryLibrary().importClass(BigInteger.class); + + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? BigInteger.valueOf(ii) : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", BigInteger.class); + + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table real_null_result = table.where("X == real_null"); + final Table null_result = table.where("X == val_null"); + Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); + Assert.eq(range_result.size(), "range_result.size()", 3); + + // let's also test BigDecimal -> BigInteger conversion; note that conversion does not round + ExecutionContext.getContext().getQueryScope().putParam("bd_5", BigDecimal.valueOf(5.8)); + final Table bd_result = table.where("X >= bd_5"); + assertTableEquals(range_result, bd_result); + } + + @Test + public void testBigDecimalCoercion() { + ExecutionContext.getContext().getQueryLibrary().importClass(BigDecimal.class); + + final Table table = emptyTable(11).update("X= ii % 2 == 0 ? BigDecimal.valueOf(ii) : null"); + final Class colType = table.getDefinition().getColumn("X").getDataType(); + Assert.eq(colType, "colType", BigDecimal.class); + + ExecutionContext.getContext().getQueryScope().putParam("real_null", null); + ExecutionContext.getContext().getQueryScope().putParam("val_null", QueryConstants.NULL_INT); + ExecutionContext.getContext().getQueryScope().putParam("val_5", 5); + + final Table real_null_result = table.where("X == real_null"); + final Table null_result = table.where("X == val_null"); + Assert.eq(null_result.size(), "null_result.size()", 5); + assertTableEquals(real_null_result, null_result); + + final Table range_result = table.where("X >= val_5"); + Assert.eq(range_result.size(), "range_result.size()", 3); + + // let's also test BigInteger -> BigDecimal conversion + ExecutionContext.getContext().getQueryScope().putParam("bi_5", BigInteger.valueOf(5)); + final Table bi_result = table.where("X >= bi_5"); + assertTableEquals(range_result, bi_result); + } } From 2ebba51d098f089c08acc2c10b63246877067ded Mon Sep 17 00:00:00 2001 From: Nathaniel Bauernfeind Date: Tue, 25 Jun 2024 09:33:52 -0600 Subject: [PATCH 44/44] revert commented tests --- .../table/impl/QueryTableWhereTest.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java index 145141b2b32..acfa9686b6f 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereTest.java @@ -387,31 +387,31 @@ public void testWhereDynamicIn() { }); } - // @Test - // public void testWhereDynamicInIncremental() { - // testWhereDynamicIncrementalInternal(false, false, true, true); - // testWhereDynamicIncrementalInternal(false, false, true, false); - // testWhereDynamicIncrementalInternal(false, false, false, true); - // testWhereDynamicIncrementalInternal(false, false, false, false); - // } - // - // @Test - // public void testWhereDynamicInIncrementalIndexed() { - // testWhereDynamicIncrementalInternal(true, false, true, true); - // testWhereDynamicIncrementalInternal(true, false, true, false); - // testWhereDynamicIncrementalInternal(true, false, false, true); - // testWhereDynamicIncrementalInternal(true, false, false, false); - // - // testWhereDynamicIncrementalInternal(false, true, true, true); - // testWhereDynamicIncrementalInternal(false, true, true, false); - // testWhereDynamicIncrementalInternal(false, true, false, true); - // testWhereDynamicIncrementalInternal(false, true, false, false); - // - // testWhereDynamicIncrementalInternal(true, true, true, true); - // testWhereDynamicIncrementalInternal(true, true, true, false); - // testWhereDynamicIncrementalInternal(true, true, false, true); - // testWhereDynamicIncrementalInternal(true, true, false, false); - // } + @Test + public void testWhereDynamicInIncremental() { + testWhereDynamicIncrementalInternal(false, false, true, true); + testWhereDynamicIncrementalInternal(false, false, true, false); + testWhereDynamicIncrementalInternal(false, false, false, true); + testWhereDynamicIncrementalInternal(false, false, false, false); + } + + @Test + public void testWhereDynamicInIncrementalIndexed() { + testWhereDynamicIncrementalInternal(true, false, true, true); + testWhereDynamicIncrementalInternal(true, false, true, false); + testWhereDynamicIncrementalInternal(true, false, false, true); + testWhereDynamicIncrementalInternal(true, false, false, false); + + testWhereDynamicIncrementalInternal(false, true, true, true); + testWhereDynamicIncrementalInternal(false, true, true, false); + testWhereDynamicIncrementalInternal(false, true, false, true); + testWhereDynamicIncrementalInternal(false, true, false, false); + + testWhereDynamicIncrementalInternal(true, true, true, true); + testWhereDynamicIncrementalInternal(true, true, true, false); + testWhereDynamicIncrementalInternal(true, true, false, true); + testWhereDynamicIncrementalInternal(true, true, false, false); + } private static void testWhereDynamicIncrementalInternal( boolean filterIndexed,