From 06147f6855fee352000f3d9655f383a05f45d83d Mon Sep 17 00:00:00 2001 From: Jiefan Li Date: Mon, 2 May 2022 14:18:54 -0700 Subject: [PATCH 1/4] Move DaliOperatorTable to Coral-Common and add abstract class FunctionResolver --- .../common/functions/CoralOperatorTable.java | 17 +++++------ .../common/functions/FunctionResolver.java | 30 +++++++++++++++++++ .../hive/hive2rel/HiveToRelConverter.java | 3 +- .../functions/HiveFunctionResolver.java | 9 +++--- .../trino/trino2rel/TrinoToRelConverter.java | 17 +++++++---- 5 files changed, 55 insertions(+), 21 deletions(-) rename coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/DaliOperatorTable.java => coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java (75%) create mode 100644 coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/DaliOperatorTable.java b/coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java similarity index 75% rename from coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/DaliOperatorTable.java rename to coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java index c193eca1c..3fec94bd3 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/DaliOperatorTable.java +++ b/coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java @@ -3,7 +3,7 @@ * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ -package com.linkedin.coral.hive.hive2rel; +package com.linkedin.coral.common.functions; import java.util.Collection; import java.util.List; @@ -19,26 +19,23 @@ import org.apache.calcite.sql.validate.SqlNameMatcher; import org.apache.calcite.util.Util; -import com.linkedin.coral.common.functions.Function; -import com.linkedin.coral.hive.hive2rel.functions.HiveFunctionResolver; - /** - * Class to resolve Dali function names in SQL definition based on + * Class to resolve Coral operator names in SQL definition based on * the mapping stored in table parameters in the metastore. */ -public class DaliOperatorTable implements SqlOperatorTable { +public class CoralOperatorTable implements SqlOperatorTable { // TODO: support injection framework to inject same function resolver here and ParseTreeBuilder. // For now, we create another instance since the function registry is simple. - private final HiveFunctionResolver funcResolver; + private final FunctionResolver funcResolver; - public DaliOperatorTable(HiveFunctionResolver funcResolver) { + public CoralOperatorTable(FunctionResolver funcResolver) { this.funcResolver = funcResolver; } /** - * Resolves functions names to corresponding Calcite UDF. HiveFunctionResolver ensures that - * {@code sqlIdentifier} has function name or corresponding class name for Dali functions. All function registry + * Resolves operator names to corresponding Calcite operator. FunctionResolver ensures that + * {@code sqlIdentifier} has operator name or corresponding class name for Coral operator. All registry * lookups performed by this class are case-insensitive. * * Calcite invokes this function multiple times during analysis phase to validate SqlCall operators. This is diff --git a/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java b/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java new file mode 100644 index 000000000..188d2a256 --- /dev/null +++ b/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java @@ -0,0 +1,30 @@ +/** + * Copyright 2018-2022 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.common.functions; + +import java.util.Collection; + + +/** + * Class to resolve hive function names in SQL to Function. + */ +public abstract class FunctionResolver { + + protected final FunctionRegistry registry; + + protected FunctionResolver(FunctionRegistry registry) { + this.registry = registry; + } + + /** + * Resolves function to concrete operator case-insensitively. + * @param functionName function name to resolve + * @return list of matching Functions or empty list if there is no match + */ + public Collection resolve(String functionName) { + return registry.lookup(functionName); + } +} diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java index 8f79b24e0..6692f15ae 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java @@ -24,6 +24,7 @@ import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.ToRelConverter; +import com.linkedin.coral.common.functions.CoralOperatorTable; import com.linkedin.coral.hive.hive2rel.functions.HiveFunctionResolver; import com.linkedin.coral.hive.hive2rel.functions.StaticHiveFunctionRegistry; import com.linkedin.coral.hive.hive2rel.parsetree.ParseTreeBuilder; @@ -75,7 +76,7 @@ protected SqlValidator getSqlValidator() { @Override protected SqlOperatorTable getOperatorTable() { - return ChainedSqlOperatorTable.of(SqlStdOperatorTable.instance(), new DaliOperatorTable(functionResolver)); + return ChainedSqlOperatorTable.of(SqlStdOperatorTable.instance(), new CoralOperatorTable(functionResolver)); } @Override diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java index 2e6b8c9ec..0ee9bbae7 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java @@ -33,6 +33,7 @@ import com.linkedin.coral.common.HiveTable; import com.linkedin.coral.common.functions.Function; import com.linkedin.coral.common.functions.FunctionRegistry; +import com.linkedin.coral.common.functions.FunctionResolver; import com.linkedin.coral.common.functions.UnknownSqlFunctionException; import static com.google.common.base.Preconditions.*; @@ -43,14 +44,13 @@ /** * Class to resolve hive function names in SQL to Function. */ -public class HiveFunctionResolver { +public class HiveFunctionResolver extends FunctionResolver { - public final FunctionRegistry registry; private final ConcurrentHashMap dynamicFunctionRegistry; private final List operators; public HiveFunctionResolver(FunctionRegistry registry, ConcurrentHashMap dynamicRegistry) { - this.registry = registry; + super(registry); this.dynamicFunctionRegistry = dynamicRegistry; this.operators = new ArrayList<>(SqlStdOperatorTable.instance().getOperatorList()); operators.add(HiveRLikeOperator.REGEXP); @@ -142,8 +142,9 @@ public Function tryResolve(@Nonnull String functionName, @Nullable Table hiveTab * @param functionName function name to resolve * @return list of matching Functions or empty list if there is no match */ + @Override public Collection resolve(String functionName) { - Collection staticLookup = registry.lookup(functionName); + Collection staticLookup = super.resolve(functionName); if (!staticLookup.isEmpty()) { return staticLookup; } else { diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java index 06cba3227..1a939103c 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java @@ -5,9 +5,9 @@ */ package com.linkedin.coral.trino.trino2rel; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.plan.RelOptCluster; @@ -24,11 +24,12 @@ import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.ToRelConverter; -import com.linkedin.coral.hive.hive2rel.DaliOperatorTable; +import com.linkedin.coral.common.functions.CoralOperatorTable; +import com.linkedin.coral.common.functions.Function; +import com.linkedin.coral.common.functions.FunctionResolver; import com.linkedin.coral.hive.hive2rel.HiveConvertletTable; import com.linkedin.coral.hive.hive2rel.HiveRelBuilder; import com.linkedin.coral.hive.hive2rel.HiveSqlValidator; -import com.linkedin.coral.hive.hive2rel.functions.HiveFunctionResolver; import com.linkedin.coral.hive.hive2rel.functions.StaticHiveFunctionRegistry; import com.linkedin.coral.trino.trino2rel.parsetree.ParseTreeBuilder; import com.linkedin.coral.trino.trino2rel.parsetree.ParserVisitorContext; @@ -47,8 +48,12 @@ public class TrinoToRelConverter extends ToRelConverter { private final ParseTreeBuilder parseTreeBuilder = new ParseTreeBuilder(); private final ParserVisitorContext parserVisitorContext = new ParserVisitorContext(); - private final HiveFunctionResolver functionResolver = - new HiveFunctionResolver(new StaticHiveFunctionRegistry(), new ConcurrentHashMap<>()); + private final FunctionResolver functionResolver = new FunctionResolver(new StaticHiveFunctionRegistry()) { + @Override + public Collection resolve(String functionName) { + return super.resolve(functionName); + } + }; private final // The validator must be reused SqlValidator sqlValidator = new HiveSqlValidator(getOperatorTable(), getCalciteCatalogReader(), @@ -74,7 +79,7 @@ protected SqlValidator getSqlValidator() { @Override protected SqlOperatorTable getOperatorTable() { - return ChainedSqlOperatorTable.of(SqlStdOperatorTable.instance(), new DaliOperatorTable(functionResolver)); + return ChainedSqlOperatorTable.of(SqlStdOperatorTable.instance(), new CoralOperatorTable(functionResolver)); } @Override From 0e6eccfb703d2928462f6c531f2a2dde40bc60f9 Mon Sep 17 00:00:00 2001 From: Jiefan Li Date: Sat, 7 May 2022 17:32:48 -0700 Subject: [PATCH 2/4] Modify resolve to be an abstract method --- .../linkedin/coral/common/functions/FunctionResolver.java | 6 ++---- .../coral/hive/hive2rel/functions/HiveFunctionResolver.java | 2 +- .../linkedin/coral/trino/trino2rel/TrinoToRelConverter.java | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java b/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java index 188d2a256..3b378ee71 100644 --- a/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java +++ b/coral-common/src/main/java/com/linkedin/coral/common/functions/FunctionResolver.java @@ -9,7 +9,7 @@ /** - * Class to resolve hive function names in SQL to Function. + * Class to resolve function names in SQL to Function. */ public abstract class FunctionResolver { @@ -24,7 +24,5 @@ protected FunctionResolver(FunctionRegistry registry) { * @param functionName function name to resolve * @return list of matching Functions or empty list if there is no match */ - public Collection resolve(String functionName) { - return registry.lookup(functionName); - } + public abstract Collection resolve(String functionName); } diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java index 0ee9bbae7..1e57d7ee3 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/functions/HiveFunctionResolver.java @@ -144,7 +144,7 @@ public Function tryResolve(@Nonnull String functionName, @Nullable Table hiveTab */ @Override public Collection resolve(String functionName) { - Collection staticLookup = super.resolve(functionName); + Collection staticLookup = registry.lookup(functionName); if (!staticLookup.isEmpty()) { return staticLookup; } else { diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java index 1a939103c..3ade5e47a 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java @@ -51,7 +51,7 @@ public class TrinoToRelConverter extends ToRelConverter { private final FunctionResolver functionResolver = new FunctionResolver(new StaticHiveFunctionRegistry()) { @Override public Collection resolve(String functionName) { - return super.resolve(functionName); + return registry.lookup(functionName); } }; private final From 13ccdec58e2cfa820e55127f66884e050a1134c1 Mon Sep 17 00:00:00 2001 From: Jiefan Li Date: Sun, 5 Jun 2022 06:33:07 -0700 Subject: [PATCH 3/4] Move CoralOperatorTable --- .../coral/common/{functions => }/CoralOperatorTable.java | 5 ++++- .../com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java | 2 +- .../linkedin/coral/trino/trino2rel/TrinoToRelConverter.java | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) rename coral-common/src/main/java/com/linkedin/coral/common/{functions => }/CoralOperatorTable.java (93%) diff --git a/coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java b/coral-common/src/main/java/com/linkedin/coral/common/CoralOperatorTable.java similarity index 93% rename from coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java rename to coral-common/src/main/java/com/linkedin/coral/common/CoralOperatorTable.java index 3fec94bd3..f98aee278 100644 --- a/coral-common/src/main/java/com/linkedin/coral/common/functions/CoralOperatorTable.java +++ b/coral-common/src/main/java/com/linkedin/coral/common/CoralOperatorTable.java @@ -3,7 +3,7 @@ * Licensed under the BSD-2 Clause license. * See LICENSE in the project root for license information. */ -package com.linkedin.coral.common.functions; +package com.linkedin.coral.common; import java.util.Collection; import java.util.List; @@ -19,6 +19,9 @@ import org.apache.calcite.sql.validate.SqlNameMatcher; import org.apache.calcite.util.Util; +import com.linkedin.coral.common.functions.Function; +import com.linkedin.coral.common.functions.FunctionResolver; + /** * Class to resolve Coral operator names in SQL definition based on diff --git a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java index 6692f15ae..6c8e59826 100644 --- a/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java +++ b/coral-hive/src/main/java/com/linkedin/coral/hive/hive2rel/HiveToRelConverter.java @@ -22,9 +22,9 @@ import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.hadoop.hive.metastore.api.Table; +import com.linkedin.coral.common.CoralOperatorTable; import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.ToRelConverter; -import com.linkedin.coral.common.functions.CoralOperatorTable; import com.linkedin.coral.hive.hive2rel.functions.HiveFunctionResolver; import com.linkedin.coral.hive.hive2rel.functions.StaticHiveFunctionRegistry; import com.linkedin.coral.hive.hive2rel.parsetree.ParseTreeBuilder; diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java index 3ade5e47a..43be98b28 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java @@ -22,9 +22,9 @@ import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.hadoop.hive.metastore.api.Table; +import com.linkedin.coral.common.CoralOperatorTable; import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.ToRelConverter; -import com.linkedin.coral.common.functions.CoralOperatorTable; import com.linkedin.coral.common.functions.Function; import com.linkedin.coral.common.functions.FunctionResolver; import com.linkedin.coral.hive.hive2rel.HiveConvertletTable; From 2ed61eedd359026b9eeedbcfd71e891a3eccc7d7 Mon Sep 17 00:00:00 2001 From: Jiefan Li Date: Mon, 6 Jun 2022 11:18:17 -0700 Subject: [PATCH 4/4] Create specific TrinoFunctionResolver --- .../trino2rel/TrinoFunctionResolver.java | 32 +++++++++++++++++++ .../trino/trino2rel/TrinoToRelConverter.java | 9 +----- 2 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoFunctionResolver.java diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoFunctionResolver.java b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoFunctionResolver.java new file mode 100644 index 000000000..8478938a2 --- /dev/null +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoFunctionResolver.java @@ -0,0 +1,32 @@ +/** + * Copyright 2022 LinkedIn Corporation. All rights reserved. + * Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package com.linkedin.coral.trino.trino2rel; + +import java.util.Collection; + +import com.linkedin.coral.common.functions.Function; +import com.linkedin.coral.common.functions.FunctionRegistry; +import com.linkedin.coral.common.functions.FunctionResolver; + + +/** + * Class to resolve Trino function names in SQL to Function. + */ +public class TrinoFunctionResolver extends FunctionResolver { + protected TrinoFunctionResolver(FunctionRegistry registry) { + super(registry); + } + + /** + * Resolves function to concrete operator case-insensitively. + * @param functionName function name to resolve + * @return list of matching Functions or empty list if there is no match + */ + @Override + public Collection resolve(String functionName) { + return registry.lookup(functionName); + } +} diff --git a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java index 43be98b28..dd9b42943 100644 --- a/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java +++ b/coral-trino/src/main/java/com/linkedin/coral/trino/trino2rel/TrinoToRelConverter.java @@ -5,7 +5,6 @@ */ package com.linkedin.coral.trino.trino2rel; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -25,7 +24,6 @@ import com.linkedin.coral.common.CoralOperatorTable; import com.linkedin.coral.common.HiveMetastoreClient; import com.linkedin.coral.common.ToRelConverter; -import com.linkedin.coral.common.functions.Function; import com.linkedin.coral.common.functions.FunctionResolver; import com.linkedin.coral.hive.hive2rel.HiveConvertletTable; import com.linkedin.coral.hive.hive2rel.HiveRelBuilder; @@ -48,12 +46,7 @@ public class TrinoToRelConverter extends ToRelConverter { private final ParseTreeBuilder parseTreeBuilder = new ParseTreeBuilder(); private final ParserVisitorContext parserVisitorContext = new ParserVisitorContext(); - private final FunctionResolver functionResolver = new FunctionResolver(new StaticHiveFunctionRegistry()) { - @Override - public Collection resolve(String functionName) { - return registry.lookup(functionName); - } - }; + private final FunctionResolver functionResolver = new TrinoFunctionResolver(new StaticHiveFunctionRegistry()); private final // The validator must be reused SqlValidator sqlValidator = new HiveSqlValidator(getOperatorTable(), getCalciteCatalogReader(),