Skip to content

Commit

Permalink
Add repl ec to execute Java and scala code
Browse files Browse the repository at this point in the history
  • Loading branch information
ChengJie1053 committed Oct 11, 2023
1 parent ca87f43 commit 524117b
Show file tree
Hide file tree
Showing 28 changed files with 1,644 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope
val runType = EngineType.mapStringToEngineType(engine) match {
case EngineType.SPARK => RunType.SQL
case EngineType.HIVE => RunType.HIVE
case EngineType.REPL => RunType.REPL
case EngineType.TRINO => RunType.TRINO_SQL
case EngineType.PRESTO => RunType.PRESTO_SQL
case EngineType.NEBULA => RunType.NEBULA_SQL
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class AMConfiguration {
public static final CommonVars<String> MULTI_USER_ENGINE_TYPES =
CommonVars.apply(
"wds.linkis.multi.user.engine.types",
"jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase");
"jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase,repl");

public static final CommonVars<String> ALLOW_BATCH_KILL_ENGINE_TYPES =
CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python");
Expand Down Expand Up @@ -105,8 +105,8 @@ public class AMConfiguration {
public static String getDefaultMultiEngineUser() {
String jvmUser = Utils.getJvmUser();
return String.format(
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\", hbase:\"%s\",io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",repl:\"%s\", hbase:\"%s\",io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
}

public static boolean isMultiUserEngine(String engineType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ public class LabelCommonConfig {
public static final CommonVars<String> PYTHON_ENGINE_VERSION =
CommonVars.apply("wds.linkis.python.engine.version", "python2");

public static final CommonVars<String> REPL_ENGINE_VERSION =
CommonVars.apply("wds.linkis.repl.engine.version", "1");

public static final CommonVars<String> FILE_ENGINE_VERSION =
CommonVars.apply("wds.linkis.file.engine.version", "1.0");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.linkis.manager.label.entity.engine

import org.apache.linkis.common.utils.Logging
Expand All @@ -33,6 +33,8 @@ object EngineType extends Enumeration with Logging {

val PYTHON = Value("python")

val REPL = Value("repl")

val SHELL = Value("shell")

val JDBC = Value("jdbc")
Expand Down Expand Up @@ -95,6 +97,7 @@ object EngineType extends Enumeration with Logging {
case _ if PIPELINE.toString.equalsIgnoreCase(str) => PIPELINE
case _ if PRESTO.toString.equalsIgnoreCase(str) => PRESTO
case _ if NEBULA.toString.equalsIgnoreCase(str) => NEBULA
case _ if REPL.toString.equalsIgnoreCase(str) => REPL
case _ if FLINK.toString.equalsIgnoreCase(str) => FLINK
case _ if APPCONN.toString.equals(str) => APPCONN
case _ if SQOOP.toString.equalsIgnoreCase(str) => SQOOP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ object RunType extends Enumeration {
val HIVE = Value("hql")
val SCALA = Value("scala")
val PYTHON = Value("python")
val REPL = Value("repl")
val JAVA = Value("java")
val PYSPARK = Value("py")
val R = Value("r")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ private static void init() {
EngineType.HIVE().toString(), LabelCommonConfig.HIVE_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.PYTHON().toString(), LabelCommonConfig.PYTHON_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.REPL().toString(), LabelCommonConfig.REPL_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.IO_ENGINE_FILE().toString(),
LabelCommonConfig.FILE_ENGINE_VERSION.getValue());
Expand Down
1 change: 1 addition & 0 deletions linkis-engineconn-plugins/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
<module>elasticsearch</module>
<module>seatunnel</module>
<module>hbase</module>
<module>repl</module>
</modules>
<profiles>
<profile>
Expand Down
127 changes: 127 additions & 0 deletions linkis-engineconn-plugins/repl/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>linkis-engineplugin-repl</artifactId>

<dependencies>
<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-engineconn-plugin-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-computation-engineconn</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-storage</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-rpc</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<!-- javassist -->
<dependency>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
<version>${javassist.version}</version>
</dependency>

<!-- scala -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>

<build>

<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<skipAssembly>false</skipAssembly>
<finalName>out</finalName>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
71 changes: 71 additions & 0 deletions linkis-engineconn-plugins/repl/src/main/assembly/distribution.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.1 https://maven.apache.org/xsd/assembly-2.1.1.xsd">
<id>linkis-engineplugin-repl</id>
<formats>
<format>dir</format>
<format>zip</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>repl</baseDirectory>

<dependencySets>
<dependencySet>
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
<!-- Now, select which projects to include in this module-set. -->
<outputDirectory>/dist/${repl.version}/lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<unpack>false</unpack>
<useStrictFiltering>false</useStrictFiltering>
<useTransitiveFiltering>true</useTransitiveFiltering>

</dependencySet>
</dependencySets>

<fileSets>

<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>linkis-engineconn.properties</include>
<include>log4j2.xml</include>
</includes>
<fileMode>0777</fileMode>
<outputDirectory>dist/${repl.version}/conf</outputDirectory>
<lineEnding>unix</lineEnding>
</fileSet>

<fileSet>
<directory>${basedir}/target</directory>
<includes>
<include>*.jar</include>
</includes>
<excludes>
<exclude>*doc.jar</exclude>
</excludes>
<fileMode>0777</fileMode>
<outputDirectory>plugin/${repl.version}</outputDirectory>
</fileSet>

</fileSets>

</assembly>

Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.linkis.engineplugin.repl;

import org.apache.linkis.engineplugin.repl.builder.ReplProcessEngineConnLaunchBuilder;
import org.apache.linkis.engineplugin.repl.factory.ReplEngineConnFactory;
import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin;
import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory;
import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder;
import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory;
import org.apache.linkis.manager.engineplugin.common.resource.GenericEngineResourceFactory;
import org.apache.linkis.manager.label.entity.Label;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class ReplEngineConnPlugin implements EngineConnPlugin {
private Object resourceLocker = new Object();
private Object engineFactoryLocker = new Object();
private volatile EngineResourceFactory engineResourceFactory;
private volatile EngineConnFactory engineFactory;
private List<Label<?>> defaultLabels = new ArrayList<>();

@Override
public void init(Map<String, Object> params) {}

@Override
public EngineResourceFactory getEngineResourceFactory() {
if (null == engineResourceFactory) {
synchronized (resourceLocker) {
engineResourceFactory = new GenericEngineResourceFactory();
}
}
return engineResourceFactory;
}

@Override
public EngineConnLaunchBuilder getEngineConnLaunchBuilder() {
return new ReplProcessEngineConnLaunchBuilder();
}

@Override
public EngineConnFactory getEngineConnFactory() {
if (null == engineFactory) {
synchronized (engineFactoryLocker) {
engineFactory = new ReplEngineConnFactory();
}
}
return engineFactory;
}

@Override
public List<Label<?>> getDefaultLabels() {
return defaultLabels;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.linkis.engineplugin.repl.builder;

import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder;

public class ReplProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {}
Loading

0 comments on commit 524117b

Please sign in to comment.