Update release artifacts (#75)

* Add hoptimator-jdbc-driver and hoptimator-api release artifacts

* Fix non-determinism in integration tests

* Move Source, Sink, Job, Pipeline to API

* Upgrade gradle

Makefile

@@ -10,7 +10,10 @@ build:
 bounce: build undeploy deploy deploy-samples deploy-config deploy-demo
 
 # Integration tests expect K8s and Kafka to be running
-integration-tests:
+integration-tests: deploy-dev-environment deploy-samples
+	kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka
+	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-1 --for=condition=Ready --timeout=10m -n kafka
+	kubectl wait kafkatopic.kafka.strimzi.io/existing-topic-2 --for=condition=Ready --timeout=10m -n kafka
 	kubectl port-forward -n kafka svc/one-kafka-external-0 9092 & echo $$! > port-forward.pid
 	./gradlew intTest || kill `cat port-forward.pid`
 	kill `cat port-forward.pid`
@@ -50,8 +53,8 @@ deploy-dev-environment: deploy-config
 	kubectl apply -f ./deploy/samples/kafkadb.yaml
 
 undeploy-dev-environment:
-	kubectl delete $(shell kubectl get kafkatopic.kafka.strimzi.io -o name -n kafka) -n kafka || echo "skipping"
-	kubectl delete $(shell kubectl get strimzi -o name -n kafka) -n kafka || echo "skipping"
+	kubectl delete kafkatopic.kafka.strimzi.io -n kafka --all || echo "skipping"
+	kubectl delete strimzi -n kafka --all || echo "skipping"
 	kubectl delete pvc -l strimzi.io/name=one-kafka -n kafka || echo "skipping"
 	kubectl delete -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka || echo "skipping"
 	kubectl delete -f ./deploy/samples/kafkadb.yaml || echo "skipping"

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -15,6 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# SPDX-License-Identifier: Apache-2.0
+#
 
 ##############################################################################
 #
@@ -55,7 +57,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -83,10 +85,9 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,26 +134,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +201,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

gradlew.bat

@@ -13,6 +13,8 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 @rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
 
 @if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
@@ -43,11 +45,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 

hoptimator-api/build.gradle

@@ -1,7 +1,53 @@
 plugins {
   id 'java'
+  id 'maven-publish'
 }
 
 dependencies {
   // plz keep it this way
 }
+
+publishing {
+  repositories {
+    maven {
+      name 'GitHubPackages'
+      url = 'https://maven.pkg.github.com/linkedin/Hoptimator'
+      credentials {
+        username = System.getenv('GITHUB_ACTOR')
+        password = System.getenv('GITHUB_TOKEN')
+      }
+    }
+    maven {
+      name 'LinkedInJFrog'
+      url 'https://linkedin.jfrog.io/artifactory/hoptimator'
+      credentials {
+        username = System.getenv('JFROG_USERNAME')
+        password = System.getenv('JFROG_API_KEY')
+      }
+    }
+  }
+  publications {
+    maven(MavenPublication) {
+      groupId = 'com.linkedin.hoptimator'
+      artifactId = 'hoptimator-api'
+      version = System.getenv('VERSION')
+      from components.java
+      pom {
+        name = 'hoptimator-api'
+        description = 'API for extending Hoptimator'
+        url = 'https://github.com/linkedin/Hoptimator'
+        licenses {
+          license {
+            name = 'BSD 2-Clause'
+            url = 'https://raw.githubusercontent.com/linkedin/Hoptimator/main/LICENSE'
+            }
+          }
+        scm {
+          connection = 'scm:git:git://github.com:linkedin/Hoptimator.git'
+          developerConnection = 'scm:git:ssh://github.com:linkedin/Hoptimator.git'
+          url = 'https://github.com/linkedin/Hoptimator'
+        }
+      }
+    }
+  }
+}

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/Job.java → hoptimator-api/src/main/java/com/linkedin/hoptimator/Job.java

@@ -1,9 +1,7 @@
-package com.linkedin.hoptimator.util;
+package com.linkedin.hoptimator;
 
 import java.util.function.Function;
 
-import org.apache.calcite.sql.SqlDialect;
-
 
 public class Job {
 

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/MaterializedView.java → hoptimator-api/src/main/java/com/linkedin/hoptimator/MaterializedView.java

@@ -1,13 +1,9 @@
-package com.linkedin.hoptimator.util;
+package com.linkedin.hoptimator;
 
 import java.util.List;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import org.apache.calcite.sql.SqlDialect;
-
-import com.linkedin.hoptimator.util.planner.Pipeline;
-
 
 public class MaterializedView {
 

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/planner/Pipeline.java → hoptimator-api/src/main/java/com/linkedin/hoptimator/Pipeline.java

@@ -1,11 +1,9 @@
-package com.linkedin.hoptimator.util.planner;
+package com.linkedin.hoptimator;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
 
-import com.linkedin.hoptimator.Deployable;
-
 
 /**
  * A set of Deployable objects that work together to deliver data.

hoptimator-api/src/main/java/com/linkedin/hoptimator/Sink.java

@@ -0,0 +1,17 @@
+package com.linkedin.hoptimator;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class Sink extends Source {
+
+  public Sink(String database, List<String> path, Map<String, String> options) {
+    super(database, path, options);
+  }
+
+  @Override
+  public String toString() {
+    return "Sink[" + pathString() + "]";
+  }
+}

hoptimator-util/src/main/java/com/linkedin/hoptimator/util/Source.java → hoptimator-api/src/main/java/com/linkedin/hoptimator/Source.java

@@ -1,23 +1,19 @@
-package com.linkedin.hoptimator.util;
+package com.linkedin.hoptimator;
 
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.apache.calcite.rel.type.RelDataType;
-
 
 public class Source {
 
   private final String database;
   private final List<String> path;
-  private final RelDataType rowType;
   private final Map<String, String> options;
 
-  public Source(String database, List<String> path, RelDataType rowType, Map<String, String> options) {
+  public Source(String database, List<String> path, Map<String, String> options) {
     this.database = database;
     this.path = path;
-    this.rowType = rowType;
     this.options = options;
   }
 
@@ -42,10 +38,6 @@ public List<String> path() {
     return path;
   }
 
-  public RelDataType rowType() {
-    return rowType;
-  }
-
   protected String pathString() {
     return path.stream().collect(Collectors.joining("."));
   }

hoptimator-api/src/main/java/com/linkedin/hoptimator/SqlDialect.java

@@ -0,0 +1,7 @@
+package com.linkedin.hoptimator;
+
+
+public enum SqlDialect {
+  ANSI,
+  FLINK
+}

hoptimator-avro/build.gradle

@@ -2,8 +2,6 @@ plugins {
   id 'java'
 }
 
-
-
 dependencies {
   implementation project(':hoptimator-api')
   implementation libs.avro

hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java

@@ -9,9 +9,9 @@
 
 import org.apache.calcite.jdbc.CalciteConnection;
 import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.sql.dialect.AnsiSqlDialect;
 import org.jline.reader.Completer;
 
+import com.linkedin.hoptimator.SqlDialect;
 import com.linkedin.hoptimator.jdbc.HoptimatorDriver;
 import com.linkedin.hoptimator.util.DeploymentService;
 import com.linkedin.hoptimator.util.planner.PipelineRel;
@@ -89,7 +89,7 @@ public void execute(String line, DispatchCallback dispatchCallback) {
       try {
         RelNode rel = HoptimatorDriver.convert(conn.createPrepareContext(), sql).root.rel;
         PipelineRel.Implementor plan = DeploymentService.plan(rel);
-        sqlline.output(plan.sql().apply(AnsiSqlDialect.DEFAULT));
+        sqlline.output(plan.sql().apply(SqlDialect.ANSI));
       } catch (SQLException e) {
         sqlline.error(e);
         dispatchCallback.setToFailure();