From bd0e25ed7ea3d012785760b82436cc3f41809541 Mon Sep 17 00:00:00 2001 From: Tobias Kleinert Date: Fri, 12 Jul 2024 13:35:31 +0200 Subject: [PATCH 01/16] add graalpy gradle plugin --- graalpython/graalpy-gradle-plugin/pom.xml | 188 +++++++++ .../graalvm/python/GraalPyGradlePlugin.java | 103 +++++ .../java/org/graalvm/python/GradleLogger.java | 37 ++ .../graalvm/python/dsl/GraalPyExtension.java | 23 ++ .../graalvm/python/dsl/PythonHomeInfo.java | 8 + .../python/tasks/GenerateManifestTask.java | 58 +++ .../python/tasks/InstallPackagesTask.java | 358 ++++++++++++++++++ .../org.graalvm.python.properties | 3 + mx.graalpython/suite.py | 12 + 9 files changed, 790 insertions(+) create mode 100644 graalpython/graalpy-gradle-plugin/pom.xml create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties diff --git a/graalpython/graalpy-gradle-plugin/pom.xml b/graalpython/graalpy-gradle-plugin/pom.xml new file mode 100644 index 0000000000..180853bf8d --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/pom.xml @@ -0,0 +1,188 @@ + + + + + 4.0.0 + + org.graalvm.python + graalpy-gradle-plugin + jar + + 24.2.0 + http://www.graalvm.org/ + graalpy-gradle-plugin + Handles python related resources in a gradle GraalPy - Java application. + + + 17 + 17 + UTF-8 + 24.2.0 + 6.1.1 + + + + + Version from suite + + + env.GRAALPY_VERSION + + + + ${env.GRAALPY_VERSION} + + + + + + + + org.codehaus.groovy + groovy + 3.0.22 + provided + + + org.gradle + gradle-tooling-api + 8.8 + + + org.gradle + gradle-core + provided + ${gradle.version} + + + org.gradle + gradle-core-api + provided + ${gradle.version} + + + org.gradle + gradle-model-core + provided + ${gradle.version} + + + org.gradle + gradle-language-java + provided + ${gradle.version} + + + org.gradle + gradle-language-jvm + provided + ${gradle.version} + + + org.gradle + gradle-platform-jvm + provided + ${gradle.version} + + + org.gradle + gradle-base-services + ${gradle.version} + provided + + + org.gradle + gradle-plugins + provided + ${gradle.version} + + + org.gradle + gradle-logging + ${gradle.version} + provided + + + + org.graalvm.python + python-embedding-tools + ${graalpy.version} + compile + + + org.graalvm.python + python-launcher + ${graalpy.version} + runtime + + + + + + maven + https://repo1.maven.org/maven2/ + + + gradle + https://repo.gradle.org/gradle/libs-releases/ + + true + + + false + + + + gradle-local + https://repo.gradle.org/gradle/libs-releases-local/ + + true + + + false + + + + + + diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java new file mode 100644 index 0000000000..219ab7b311 --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -0,0 +1,103 @@ +package org.graalvm.python; + +import org.graalvm.python.dsl.GraalPyExtension; +import org.graalvm.python.tasks.GenerateManifestTask; +import org.graalvm.python.tasks.InstallPackagesTask; +import org.gradle.api.GradleException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.DependencySet; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.jvm.tasks.Jar; +import org.gradle.language.jvm.tasks.ProcessResources; + +import java.util.Collections; +import java.util.List; + + +public abstract class GraalPyGradlePlugin implements Plugin { + private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; + private static final String PYTHON_EMBEDDING_ARTIFACT_ID = "python-embedding"; + public static final String GRAALPY_GROUP_ID = "org.graalvm.python"; + + private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; + private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; + private static final String PYTHON_ARTIFACT_ID = "python"; + private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalpy"; + + private static final String DEFAULT_WRAPPER_DIRECTORY = "python-generated"; + + GraalPyExtension extension; + Project project; + + @Override + public void apply(Project project) { + this.project = project; + project.getPluginManager().apply(JavaPlugin.class); + + + this.extension = project.getExtensions().create("graalPy", GraalPyExtension.class); + extension.getPythonHome().getIncludes().convention(List.of(".*")); + extension.getPythonHome().getExcludes().convention(Collections.emptyList()); + extension.getPackages().convention(Collections.emptyList()); + + var installPackagesTask = project.getTasks().register("installPackages", InstallPackagesTask.class); + + + final var generateManifestTask = project.getTasks().register("generateManifest", GenerateManifestTask.class); + project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(generateManifestTask)); + generateManifestTask.configure(t -> { + t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir("GRAAL-META-INF")); + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + + installPackagesTask.configure(t -> { + t.getIncludes().set(extension.getPythonHome().getIncludes()); + t.getExcludes().set(extension.getPythonHome().getExcludes()); + t.getPackages().set(extension.getPackages()); + t.getIncludeVfsRoot().set(extension.getIncludeVfsRootDir()); + + t.getOutput().set(extension.getPythonResourcesDirectory()); + + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + + project.afterEvaluate(p -> { + checkAndAddDependencies(); + + if (!extension.getPythonResourcesDirectory().isPresent()) + ((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).with(project.copySpec().from(installPackagesTask)); + + // Provide the default value after the isPresent check, otherwise isPresent always returns true + extension.getPythonResourcesDirectory().convention(project.getLayout().getBuildDirectory().dir(DEFAULT_WRAPPER_DIRECTORY)); + }); + + } + + + private void checkAndAddDependencies() { + project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_LAUNCHER_ARTIFACT_ID, getGraalPyVersion(project))); + project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_EMBEDDING_ARTIFACT_ID, getGraalPyVersion(project))); + } + + + public static String getGraalPyVersion(Project project) { + return getGraalPyDependency(project).getVersion(); + } + + public static Dependency getGraalPyDependency(Project project) { + return resolveProjectDependencies(project).stream().filter(GraalPyGradlePlugin::isPythonArtifact).findFirst().orElseThrow(() -> new GradleException("Missing GraalPy dependency. Please add to your build.gradle either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); + } + + private static boolean isPythonArtifact(Dependency dependency) { + return (POLYGLOT_GROUP_ID.equals(dependency.getGroup()) || GRAALPY_GROUP_ID.equals(dependency.getGroup())) && + (PYTHON_COMMUNITY_ARTIFACT_ID.equals(dependency.getName()) || PYTHON_ARTIFACT_ID.equals(dependency.getName())); + } + + private static DependencySet resolveProjectDependencies(Project project) { + return project.getConfigurations().getByName("implementation").getAllDependencies(); + } + + +} \ No newline at end of file diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java new file mode 100644 index 0000000000..fa0288ea35 --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java @@ -0,0 +1,37 @@ +package org.graalvm.python; + +import groovy.util.logging.Log; +import org.graalvm.python.embedding.tools.exec.SubprocessLog; +import org.gradle.api.logging.Logger; + +public class GradleLogger implements SubprocessLog { + private Logger logger; + + private GradleLogger(Logger logger) { + this.logger = logger; + } + + @Override + public void subProcessOut(CharSequence out) { + logger.lifecycle(out.toString()); + } + + @Override + public void subProcessErr(CharSequence err) { + logger.warn(err.toString()); + } + + @Override + public void log(CharSequence txt) { + logger.lifecycle(txt.toString()); + } + + @Override + public void log(CharSequence txt, Throwable t) { + logger.lifecycle(txt.toString(), t); + } + + public static GradleLogger of(Logger logger) { + return new GradleLogger(logger); + } +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java new file mode 100644 index 0000000000..85422fbc3c --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java @@ -0,0 +1,23 @@ +package org.graalvm.python.dsl; + +import org.gradle.api.Action; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Nested; + +public interface GraalPyExtension { + DirectoryProperty getPythonResourcesDirectory(); + + ListProperty getPackages(); + + Property getIncludeVfsRootDir(); + + @Nested + PythonHomeInfo getPythonHome(); + + default void pythonHome(Action action) { + action.execute(getPythonHome()); + } + +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java new file mode 100644 index 0000000000..d0b285c8ba --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java @@ -0,0 +1,8 @@ +package org.graalvm.python.dsl; + +import org.gradle.api.provider.SetProperty; + +public interface PythonHomeInfo { + SetProperty getIncludes(); + SetProperty getExcludes(); +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java new file mode 100644 index 0000000000..8264a6dd5a --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java @@ -0,0 +1,58 @@ +package org.graalvm.python.tasks; + +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleScriptException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import static org.graalvm.python.GraalPyGradlePlugin.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; + + +public abstract class GenerateManifestTask extends DefaultTask { + + private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ + { + "resources": { + "includes": [ + {"pattern": "$vfs/.*"} + ] + } + } + """.replace("$vfs", VFS_ROOT); + + private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; + private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "graalpy-gradle-plugin"; + + @OutputDirectory + public abstract DirectoryProperty getManifestOutputDir(); + + @TaskAction + public void generateManifest() { + Path metaInf = getMetaInfDirectory(); + Path resourceConfig = metaInf.resolve("resource-config.json"); + try { + Files.createDirectories(resourceConfig.getParent()); + Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to write %s", resourceConfig), e); + } + Path nativeImageProperties = metaInf.resolve("native-image.properties"); + try { + Files.createDirectories(nativeImageProperties.getParent()); + Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to write %s", nativeImageProperties), e); + } + } + + private Path getMetaInfDirectory() { + return Path.of(getManifestOutputDir().get().getAsFile().getAbsolutePath(), "native-image", GRAALPY_GROUP_ID, GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); + } +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java new file mode 100644 index 0000000000..c3eaccdaed --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java @@ -0,0 +1,358 @@ +package org.graalvm.python.tasks; + +import org.graalvm.python.GradleLogger; +import org.graalvm.python.embedding.tools.exec.GraalPyRunner; +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.GradleScriptException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.*; +import org.gradle.api.tasks.*; +import org.gradle.api.tasks.Optional; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.*; +import java.util.stream.Collectors; + +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyDependency; +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyVersion; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.*; + +public abstract class InstallPackagesTask extends DefaultTask { + private static final String GRAALPY_GROUP_ID = "org.graalvm.python"; + + private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; + + private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); + private static final String LAUNCHER = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; + + private static final String INCLUDE_PREFIX = "include:"; + + private static final String EXCLUDE_PREFIX = "exclude:"; + + private Set launcherClassPath; + + @Input + @Optional + public abstract Property getIncludeVfsRoot(); + + @Input + public abstract ListProperty getPackages(); + + @Input + public abstract ListProperty getIncludes(); + + @Input + public abstract ListProperty getExcludes(); + + @OutputDirectory + public abstract DirectoryProperty getOutput(); + + @TaskAction + public void exec() { + manageHome(); + manageVenv(); + listGraalPyResources(); + } + + private void manageHome() { + Path homeDirectory = getHomeDirectory(); + Path tagfile = homeDirectory.resolve("tagfile"); + String graalPyVersion = getGraalPyVersion(getProject()); + + List includes = new ArrayList<>(getIncludes().get()); + List excludes = new ArrayList<>(getExcludes().get()); + + trim(includes); + trim(excludes); + + if (Files.isReadable(tagfile)) { + List lines; + try { + lines = Files.readAllLines(tagfile); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to read tag file %s", tagfile), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + getLogger().lifecycle(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); + delete(homeDirectory); + } + if (pythonHomeChanged(includes, excludes, lines)) { + getLogger().lifecycle(String.format("Deleting GraalPy home due to changed includes or excludes")); + delete(homeDirectory); + } + } + + try { + if (!Files.exists(homeDirectory)) { + getLogger().lifecycle(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); + Files.createDirectories(homeDirectory.getParent()); + VFSUtils.copyGraalPyHome(calculateLauncherClasspath(), homeDirectory, includes, excludes, GradleLogger.of(getLogger())); + } + Files.write(tagfile, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + write(tagfile, includes, INCLUDE_PREFIX); + write(tagfile, excludes, EXCLUDE_PREFIX); + } catch (IOException | InterruptedException e) { + throw new GradleException(String.format("failed to copy graalpy home %s", homeDirectory), e); + } + } + + private void manageVenv() { + generateLaunchers(); + + Path venvDirectory = getVenvDirectory(); + + Path venvContents = venvDirectory.resolve("contents"); + List installedPackages = new ArrayList<>(); + String graalPyVersion = getGraalPyVersion(getProject()); + + var packages = getPackages().getOrElse(null); + + if (packages != null && !packages.isEmpty()) { + trim(packages); + } + + if (packages == null || packages.isEmpty()) { + getLogger().lifecycle(String.format("No venv packages declared, deleting %s", venvDirectory)); + delete(venvDirectory); + return; + } + + if (Files.isReadable(venvContents)) { + List lines = null; + try { + lines = Files.readAllLines(venvContents); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to read tag file %s", venvContents), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + getLogger().lifecycle(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); + delete(venvDirectory); + } else { + for (int i = 1; i < lines.size(); i++) { + installedPackages.add(lines.get(i)); + } + } + } else { + getLogger().lifecycle(String.format("Creating GraalPy %s venv", graalPyVersion)); + } + + if (!Files.exists(venvDirectory)) { + runLauncher(getLauncherPath().toString(), "-m", "venv", venvDirectory.toString(), "--without-pip"); + runVenvBin(venvDirectory, "graalpy", "-I", "-m", "ensurepip"); + } + + deleteUnwantedPackages(venvDirectory, installedPackages, packages); + installWantedPackages(venvDirectory, installedPackages, packages); + + try { + Files.write(venvContents, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.write(venvContents, packages, StandardOpenOption.APPEND); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to write tag file %s", venvContents), e); + } + } + + private void listGraalPyResources() { + Path vfs = getHomeDirectory().getParent(); + if (Files.exists(vfs)) { + try { + VFSUtils.generateVFSFilesList(vfs); + } catch (IOException e) { + throw new GradleScriptException(String.format("Failed to generate files list in '%s'", vfs), e); + } + } + } + + private Set calculateLauncherClasspath() { + if (launcherClassPath == null) { + var addedPluginDependency = getProject().getConfigurations().getByName("runtimeClasspath").getAllDependencies().stream().filter(d -> d.getGroup().equals(GRAALPY_GROUP_ID) && d.getName().equals("python-launcher") && d.getVersion().equals(getGraalPyVersion(getProject()))).findFirst().orElseThrow(); + launcherClassPath = getProject().getConfigurations().getByName("runtimeClasspath").files(addedPluginDependency).stream().map(File::toString).collect(Collectors.toSet()); + launcherClassPath.addAll(getProject().getConfigurations().getByName("runtimeClasspath").files(getGraalPyDependency(getProject())).stream().map(File::toString).collect(Collectors.toSet())); + } + return launcherClassPath; + } + + private boolean pythonHomeChanged(List includes, List excludes, List lines) { + Set prevIncludes = new HashSet<>(); + Set prevExcludes = new HashSet<>(); + for (int i = 1; i < lines.size(); i++) { + String l = lines.get(i); + if (l.startsWith(INCLUDE_PREFIX)) { + prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); + } else if (l.startsWith(EXCLUDE_PREFIX)) { + prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); + } + } + boolean includeDidNotChange = prevIncludes.size() == includes.size() && prevIncludes.containsAll(includes); + boolean excludeDidNotChange = prevExcludes.size() == excludes.size() && prevExcludes.containsAll(excludes); + return !(includeDidNotChange && excludeDidNotChange); + } + + private void generateLaunchers() { + getLogger().lifecycle("Generating GraalPy launchers"); + var launcher = getLauncherPath(); + if (!Files.exists(launcher)) { + var java = Paths.get(System.getProperty("java.home"), "bin", "java"); + var classpath = calculateLauncherClasspath(); + if (!IS_WINDOWS) { + var script = String.format(""" + #!/usr/bin/env bash + %s -classpath %s %s --python.Executable="$0" "$@" + """, + java, + String.join(File.pathSeparator, classpath), + GRAALPY_MAIN_CLASS); + try { + Files.createDirectories(launcher.getParent()); + Files.writeString(launcher, script); + var perms = Files.getPosixFilePermissions(launcher); + perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); + Files.setPosixFilePermissions(launcher, perms); + } catch (IOException e) { + throw new GradleException(String.format("failed to create launcher %s", launcher), e); + } + } else { + // on windows, generate a venv launcher that executes our mvn target + var script = String.format(""" + import os, shutil, struct, venv + from pathlib import Path + vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') + tl = os.path.join(r'%s') + os.makedirs(Path(tl).parent.absolute(), exist_ok=True) + shutil.copy(vl, tl) + cmd = r'%s -classpath "%s" %s' + pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") + with open(pyvenvcfg, 'w', encoding='utf-8') as f: + f.write('venvlauncher_command = ') + f.write(cmd) + """, + launcher, + java, + String.join(File.pathSeparator, classpath), + GRAALPY_MAIN_CLASS); + File tmp; + try { + tmp = File.createTempFile("create_launcher", ".py"); + } catch (IOException e) { + throw new GradleScriptException("failed to create tmp launcher", e); + } + tmp.deleteOnExit(); + try (var wr = new FileWriter(tmp)) { + wr.write(script); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to write tmp launcher %s", tmp), e); + } + runGraalPy(tmp.getAbsolutePath()); + } + } + } + + private void runGraalPy(String... args) { + var classpath = calculateLauncherClasspath(); + try { + GraalPyRunner.run(classpath, GradleLogger.of(getLogger()), args); + } catch (IOException | InterruptedException e) { + throw new GradleScriptException("failed to run Graalpy launcher", e); + } + } + + private Path getLauncherPath() { + return Paths.get(getProject().getBuildDir().getAbsolutePath(), LAUNCHER); + } + + + private static void write(Path tag, Collection list, String prefix) throws IOException { + if (list != null && !list.isEmpty()) { + Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); + } + } + + private Path getHomeDirectory() { + return getResourceDirectory(VFS_HOME); + } + + private Path getVenvDirectory() { + return getResourceDirectory(VFS_VENV); + } + + private Path getResourceDirectory(String type) { + return Path.of(getOutput().get().getAsFile().toURI()).resolve(getIncludeVfsRoot().getOrElse(true) ? VFS_ROOT : "").resolve(type); + } + + private static void delete(Path homeDirectory) { + try { + try (var s = Files.walk(homeDirectory)) { + s.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to delete %s", homeDirectory), e); + } + } + + private static void trim(List l) { + Iterator it = l.iterator(); + while (it.hasNext()) { + String p = it.next(); + if (p == null || p.trim().isEmpty()) { + it.remove(); + } + } + } + + private void installWantedPackages(Path venvDirectory, List installedPackages, List wantedPackages) { + var pkgsToInstall = new HashSet<>(wantedPackages); + installedPackages.forEach(pkgsToInstall::remove); + if (pkgsToInstall.isEmpty()) { + return; + } + getLogger().lifecycle("Installing packages {}", pkgsToInstall); + runPip(venvDirectory, "install", pkgsToInstall.toArray(new String[pkgsToInstall.size()])); + } + + private void deleteUnwantedPackages(Path venvDirectory, List installedPackages, List wantedPackages) { + List args = new ArrayList<>(installedPackages); + args.removeAll(wantedPackages); + if (args.isEmpty()) { + return; + } + args.add(0, "-y"); + getLogger().lifecycle("Removing packages {}", args); + runPip(venvDirectory, "uninstall", args.toArray(new String[args.size()])); + } + + + private void runLauncher(String launcherPath, String... args) { + try { + GraalPyRunner.runLauncher(launcherPath, GradleLogger.of(getLogger()), args); + } catch (IOException | InterruptedException e) { + throw new GradleScriptException(String.format("failed to execute launcher command %s", List.of(args)), e); + } + } + + private void runPip(Path venvDirectory, String command, String... args) { + try { + GraalPyRunner.runPip(venvDirectory, command, GradleLogger.of(getLogger()), args); + } catch (IOException | InterruptedException e) { + throw new GradleScriptException(String.format("failed to execute pip", args), e); + } + } + + private void runVenvBin(Path venvDirectory, String bin, String... args) { + try { + GraalPyRunner.runVenvBin(venvDirectory, bin, GradleLogger.of(getLogger()), args); + } catch (IOException | InterruptedException e) { + throw new GradleScriptException(String.format("failed to execute venv", args), e); + } + } +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties b/graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties new file mode 100644 index 0000000000..2b133dff16 --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties @@ -0,0 +1,3 @@ +implementation-class=org.graalvm.python.GraalPyGradlePlugin +version=24.2.0 +id=org.graalvm.python diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index dbb644e312..702b778d59 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -1466,5 +1466,17 @@ "tag": ["default", "public"], }, }, + "graalpy-gradle-plugin": { + "class": "MavenProject", + "subDir": "graalpython", + "noMavenJavadoc": True, + "dependencies": [ + "GRAALPYTHON-LAUNCHER", + "GRAALPYTHON_EMBEDDING_TOOLS", + ], + "maven": { + "tag": ["default", "public"], + }, + }, }, } From 815a0e667d15886cd60bcdaecbe4fe77940bbc73 Mon Sep 17 00:00:00 2001 From: Tobias Kleinert Date: Fri, 12 Jul 2024 13:44:16 +0200 Subject: [PATCH 02/16] add license --- .../graalvm/python/GraalPyGradlePlugin.java | 40 +++++++++++++++++++ .../java/org/graalvm/python/GradleLogger.java | 40 +++++++++++++++++++ .../graalvm/python/dsl/GraalPyExtension.java | 40 +++++++++++++++++++ .../graalvm/python/dsl/PythonHomeInfo.java | 40 +++++++++++++++++++ .../python/tasks/GenerateManifestTask.java | 40 +++++++++++++++++++ .../python/tasks/InstallPackagesTask.java | 40 +++++++++++++++++++ 6 files changed, 240 insertions(+) diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index 219ab7b311..ebf9658ce0 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python; import org.graalvm.python.dsl.GraalPyExtension; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java index fa0288ea35..b6027e4b01 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python; import groovy.util.logging.Log; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java index 85422fbc3c..6dec18ad75 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python.dsl; import org.gradle.api.Action; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java index d0b285c8ba..bd574c2bca 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python.dsl; import org.gradle.api.provider.SetProperty; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java index 8264a6dd5a..bd302103dd 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python.tasks; import org.gradle.api.DefaultTask; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java index c3eaccdaed..bd0cd35fa4 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java @@ -1,3 +1,43 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ package org.graalvm.python.tasks; import org.graalvm.python.GradleLogger; From 7edb41ad371f249dafa8d8b310897cca46f09917 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Mon, 15 Jul 2024 18:00:43 +0200 Subject: [PATCH 03/16] extracted generating of native image config files from maven and gradle plugin into embedding.tools --- .gitignore | 2 +- graalpython/graalpy-gradle-plugin/pom.xml | 9 +++-- .../graalvm/python/GraalPyGradlePlugin.java | 3 +- .../python/tasks/GenerateManifestTask.java | 38 ++----------------- .../maven/plugin/ManageResourcesMojo.java | 32 ++-------------- .../python/embedding/tools/vfs/VFSUtils.java | 33 ++++++++++++++++ 6 files changed, 47 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index 571c47c868..e1851289c1 100644 --- a/.gitignore +++ b/.gitignore @@ -87,8 +87,8 @@ graalpython/com.oracle.graal.python.test/src/tests/patched_package/build/ graalpython/com.oracle.graal.python.test/src/tests/patched_package/src/patched_package.egg-info graalpython/com.oracle.graal.python.test.integration/target graalpython/graalpy-maven-plugin/target +graalpython/graalpy-gradle-plugin/target graalpython/graalpy-archetype-polyglot-app/target -graalpython/graalpy-micronaut-embedding/target graalpython/com.oracle.graal.python.test/src/tests/standalone/micronaut/hello/target/ graalpython/com.oracle.graal.python.test/src/tests/cpyext/build/ pom-mx.xml diff --git a/graalpython/graalpy-gradle-plugin/pom.xml b/graalpython/graalpy-gradle-plugin/pom.xml index 180853bf8d..d34f1ff87e 100644 --- a/graalpython/graalpy-gradle-plugin/pom.xml +++ b/graalpython/graalpy-gradle-plugin/pom.xml @@ -57,6 +57,7 @@ SOFTWARE. 17 17 UTF-8 + 24.2.0 6.1.1 @@ -158,10 +159,10 @@ SOFTWARE. - - maven - https://repo1.maven.org/maven2/ - + + + + gradle https://repo.gradle.org/gradle/libs-releases/ diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index ebf9658ce0..d349a0dcb4 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -55,11 +55,12 @@ import java.util.Collections; import java.util.List; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; + public abstract class GraalPyGradlePlugin implements Plugin { private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; private static final String PYTHON_EMBEDDING_ARTIFACT_ID = "python-embedding"; - public static final String GRAALPY_GROUP_ID = "org.graalvm.python"; private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java index bd302103dd..1e801af8fe 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java @@ -40,34 +40,16 @@ */ package org.graalvm.python.tasks; +import org.graalvm.python.embedding.tools.vfs.VFSUtils; import org.gradle.api.DefaultTask; import org.gradle.api.GradleScriptException; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.tasks.OutputDirectory; import org.gradle.api.tasks.TaskAction; - import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -import static org.graalvm.python.GraalPyGradlePlugin.GRAALPY_GROUP_ID; -import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; - public abstract class GenerateManifestTask extends DefaultTask { - private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ - { - "resources": { - "includes": [ - {"pattern": "$vfs/.*"} - ] - } - } - """.replace("$vfs", VFS_ROOT); - - private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "graalpy-gradle-plugin"; @OutputDirectory @@ -75,24 +57,10 @@ public abstract class GenerateManifestTask extends DefaultTask { @TaskAction public void generateManifest() { - Path metaInf = getMetaInfDirectory(); - Path resourceConfig = metaInf.resolve("resource-config.json"); try { - Files.createDirectories(resourceConfig.getParent()); - Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + VFSUtils.writeNativeImageConfig(getManifestOutputDir().get().getAsFile().getAbsolutePath(), GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); } catch (IOException e) { - throw new GradleScriptException(String.format("failed to write %s", resourceConfig), e); + throw new GradleScriptException(e.getMessage(), e); } - Path nativeImageProperties = metaInf.resolve("native-image.properties"); - try { - Files.createDirectories(nativeImageProperties.getParent()); - Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to write %s", nativeImageProperties), e); - } - } - - private Path getMetaInfDirectory() { - return Path.of(getManifestOutputDir().get().getAsFile().getAbsolutePath(), "native-image", GRAALPY_GROUP_ID, GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); } } diff --git a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java index f348638cfc..e5c5d32b04 100644 --- a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java +++ b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java @@ -69,6 +69,7 @@ import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_HOME; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_VENV; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; @Mojo(name = "process-graalpy-resources", defaultPhase = LifecyclePhase.PROCESS_RESOURCES, requiresDependencyCollection = ResolutionScope.COMPILE_PLUS_RUNTIME, @@ -78,7 +79,6 @@ public class ManageResourcesMojo extends AbstractMojo { private static final String PYTHON_LANGUAGE_ARTIFACT_ID = "python-language"; private static final String PYTHON_RESOURCES = "python-resources"; private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; - private static final String GRAALPY_GROUP_ID = "org.graalvm.python"; private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; @@ -94,18 +94,6 @@ public class ManageResourcesMojo extends AbstractMojo { private static final String EXCLUDE_PREFIX = "exclude:"; - private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ - { - "resources": { - "includes": [ - {"pattern": "$vfs/.*"} - ] - } - } - """.replace("$vfs", VFS_ROOT); - - private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; - @Parameter(defaultValue = "${project}", required = true, readonly = true) MavenProject project; @@ -131,10 +119,6 @@ public static class PythonHome { private Set launcherClassPath; - static Path getMetaInfDirectory(MavenProject project) { - return Path.of(project.getBuild().getOutputDirectory(), "META-INF", "native-image", GRAALPY_GROUP_ID, GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); - } - public void execute() throws MojoExecutionException { if(pythonResourcesDirectory != null) { @@ -177,20 +161,10 @@ private void trim(List l) { } private void manageNativeImageConfig() throws MojoExecutionException { - Path metaInf = getMetaInfDirectory(project); - Path resourceConfig = metaInf.resolve("resource-config.json"); - try { - Files.createDirectories(resourceConfig.getParent()); - Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write %s", resourceConfig), e); - } - Path nativeImageProperties = metaInf.resolve("native-image.properties"); try { - Files.createDirectories(nativeImageProperties.getParent()); - Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + VFSUtils.writeNativeImageConfig(project.getBuild().getOutputDirectory(), GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write %s", nativeImageProperties), e); + throw new MojoExecutionException(e); } } diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java index 4fa7b69ea5..d47604387b 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java @@ -51,6 +51,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; @@ -69,6 +70,38 @@ public final class VFSUtils { public static final String VFS_VENV = "venv"; public static final String VFS_FILESLIST = "fileslist.txt"; + public static final String GRAALPY_GROUP_ID = "org.graalvm.python"; + + private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ + { + "resources": { + "includes": [ + {"pattern": "$vfs/.*"} + ] + } + } + """.replace("$vfs", VFS_ROOT); + + private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; + + public static void writeNativeImageConfig(String metaInfRoot, String pluginId) throws IOException { + Path metaInf = Path.of(metaInfRoot, "META-INF", "native-image", GRAALPY_GROUP_ID, pluginId); + Path resourceConfig = metaInf.resolve("resource-config.json"); + try { + Files.createDirectories(resourceConfig.getParent()); + Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new IOException(String.format("failed to write %s", resourceConfig), e); + } + Path nativeImageProperties = metaInf.resolve("native-image.properties"); + try { + Files.createDirectories(nativeImageProperties.getParent()); + Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new IOException(String.format("failed to write %s", nativeImageProperties), e); + } + } + public static void generateVFSFilesList(Path vfs) throws IOException { Path filesList = vfs.resolve(VFS_FILESLIST); if (!Files.isDirectory(vfs)) { From 31cfe5e0b61ccce07bd3c511414ab4d40a67ffc2 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Tue, 16 Jul 2024 18:46:33 +0200 Subject: [PATCH 04/16] extracted home and venv logic from maven and gradle plugin into embedding.tools --- graalpython/graalpy-gradle-plugin/pom.xml | 15 - .../graalvm/python/GraalPyGradlePlugin.java | 44 +- .../python/tasks/InstallPackagesTask.java | 398 ------------------ ...stTask.java => NativeImageConfigTask.java} | 6 +- .../graalvm/python/tasks/ResourcesTask.java | 141 +++++++ .../python/tasks/VFSFilesListTask.java | 78 ++++ .../maven/plugin/ManageResourcesMojo.java | 290 ++----------- .../embedding/tools/exec/GraalPyRunner.java | 6 +- .../python/embedding/tools/vfs/VFSUtils.java | 288 ++++++++++++- 9 files changed, 565 insertions(+), 701 deletions(-) delete mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java rename graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/{GenerateManifestTask.java => NativeImageConfigTask.java} (93%) create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java create mode 100644 graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java diff --git a/graalpython/graalpy-gradle-plugin/pom.xml b/graalpython/graalpy-gradle-plugin/pom.xml index d34f1ff87e..e5daa9a08a 100644 --- a/graalpython/graalpy-gradle-plugin/pom.xml +++ b/graalpython/graalpy-gradle-plugin/pom.xml @@ -57,7 +57,6 @@ SOFTWARE. 17 17 UTF-8 - 24.2.0 6.1.1 @@ -159,10 +158,6 @@ SOFTWARE. - - - - gradle https://repo.gradle.org/gradle/libs-releases/ @@ -173,16 +168,6 @@ SOFTWARE. false - - gradle-local - https://repo.gradle.org/gradle/libs-releases-local/ - - true - - - false - - diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index d349a0dcb4..3112db0f82 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -41,14 +41,16 @@ package org.graalvm.python; import org.graalvm.python.dsl.GraalPyExtension; -import org.graalvm.python.tasks.GenerateManifestTask; -import org.graalvm.python.tasks.InstallPackagesTask; +import org.graalvm.python.tasks.VFSFilesListTask; +import org.graalvm.python.tasks.NativeImageConfigTask; +import org.graalvm.python.tasks.ResourcesTask; import org.gradle.api.GradleException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.DependencySet; import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.TaskProvider; import org.gradle.jvm.tasks.Jar; import org.gradle.language.jvm.tasks.ProcessResources; @@ -61,13 +63,15 @@ public abstract class GraalPyGradlePlugin implements Plugin { private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; private static final String PYTHON_EMBEDDING_ARTIFACT_ID = "python-embedding"; - private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; private static final String PYTHON_ARTIFACT_ID = "python"; - private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalpy"; - + private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalPy"; private static final String DEFAULT_WRAPPER_DIRECTORY = "python-generated"; + private static final String GRAALPY_RESOURCES_TASK = "graalPyResources"; + private static final String GRAALPY_NATIVE_IMAGE_CONFIG_TASK = "graalPyNativeImageConfig"; + private static final String GRAALPY_VFS_FILESLIST_TASK = "graalPyVFSFilesList"; + GraalPyExtension extension; Project project; @@ -77,22 +81,12 @@ public void apply(Project project) { this.project = project; project.getPluginManager().apply(JavaPlugin.class); - this.extension = project.getExtensions().create("graalPy", GraalPyExtension.class); extension.getPythonHome().getIncludes().convention(List.of(".*")); extension.getPythonHome().getExcludes().convention(Collections.emptyList()); extension.getPackages().convention(Collections.emptyList()); - var installPackagesTask = project.getTasks().register("installPackages", InstallPackagesTask.class); - - - final var generateManifestTask = project.getTasks().register("generateManifest", GenerateManifestTask.class); - project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(generateManifestTask)); - generateManifestTask.configure(t -> { - t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir("GRAAL-META-INF")); - t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); - }); - + TaskProvider installPackagesTask = project.getTasks().register(GRAALPY_RESOURCES_TASK, ResourcesTask.class); installPackagesTask.configure(t -> { t.getIncludes().set(extension.getPythonHome().getIncludes()); t.getExcludes().set(extension.getPythonHome().getExcludes()); @@ -104,11 +98,26 @@ public void apply(Project project) { t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); + TaskProvider generateManifestTask = project.getTasks().register(GRAALPY_NATIVE_IMAGE_CONFIG_TASK, NativeImageConfigTask.class); + project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(generateManifestTask)); + generateManifestTask.configure(t -> { + t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir("GRAAL-META-INF")); + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + + TaskProvider generateVFSFilesListTask = project.getTasks().register(GRAALPY_VFS_FILESLIST_TASK, VFSFilesListTask.class); + generateVFSFilesListTask.configure(t -> { + t.getResourcesDir().convention((((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).getDestinationDir())); + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, t -> t.finalizedBy(GRAALPY_VFS_FILESLIST_TASK)); + project.afterEvaluate(p -> { checkAndAddDependencies(); - if (!extension.getPythonResourcesDirectory().isPresent()) + if (!extension.getPythonResourcesDirectory().isPresent()) { ((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).with(project.copySpec().from(installPackagesTask)); + } // Provide the default value after the isPresent check, otherwise isPresent always returns true extension.getPythonResourcesDirectory().convention(project.getLayout().getBuildDirectory().dir(DEFAULT_WRAPPER_DIRECTORY)); @@ -116,7 +125,6 @@ public void apply(Project project) { } - private void checkAndAddDependencies() { project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_LAUNCHER_ARTIFACT_ID, getGraalPyVersion(project))); project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_EMBEDDING_ARTIFACT_ID, getGraalPyVersion(project))); diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java deleted file mode 100644 index bd0cd35fa4..0000000000 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/InstallPackagesTask.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * The Universal Permissive License (UPL), Version 1.0 - * - * Subject to the condition set forth below, permission is hereby granted to any - * person obtaining a copy of this software, associated documentation and/or - * data (collectively the "Software"), free of charge and under any and all - * copyright rights in the Software, and any and all patent rights owned or - * freely licensable by each licensor hereunder covering either (i) the - * unmodified Software as contributed to or provided by such licensor, or (ii) - * the Larger Works (as defined below), to deal in both - * - * (a) the Software, and - * - * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if - * one is included with the Software each a "Larger Work" to which the Software - * is contributed by such licensors), - * - * without restriction, including without limitation the rights to copy, create - * derivative works of, display, perform, and distribute the Software and make, - * use, sell, offer for sale, import, export, have made, and have sold the - * Software and the Larger Work(s), and to sublicense the foregoing rights on - * either these or other terms. - * - * This license is subject to the following condition: - * - * The above copyright notice and either this complete permission notice or at a - * minimum a reference to the UPL must be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package org.graalvm.python.tasks; - -import org.graalvm.python.GradleLogger; -import org.graalvm.python.embedding.tools.exec.GraalPyRunner; -import org.graalvm.python.embedding.tools.vfs.VFSUtils; -import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; -import org.gradle.api.GradleScriptException; -import org.gradle.api.file.DirectoryProperty; -import org.gradle.api.provider.*; -import org.gradle.api.tasks.*; -import org.gradle.api.tasks.Optional; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.util.*; -import java.util.stream.Collectors; - -import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyDependency; -import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyVersion; -import static org.graalvm.python.embedding.tools.vfs.VFSUtils.*; - -public abstract class InstallPackagesTask extends DefaultTask { - private static final String GRAALPY_GROUP_ID = "org.graalvm.python"; - - private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; - - private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); - private static final String LAUNCHER = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; - - private static final String INCLUDE_PREFIX = "include:"; - - private static final String EXCLUDE_PREFIX = "exclude:"; - - private Set launcherClassPath; - - @Input - @Optional - public abstract Property getIncludeVfsRoot(); - - @Input - public abstract ListProperty getPackages(); - - @Input - public abstract ListProperty getIncludes(); - - @Input - public abstract ListProperty getExcludes(); - - @OutputDirectory - public abstract DirectoryProperty getOutput(); - - @TaskAction - public void exec() { - manageHome(); - manageVenv(); - listGraalPyResources(); - } - - private void manageHome() { - Path homeDirectory = getHomeDirectory(); - Path tagfile = homeDirectory.resolve("tagfile"); - String graalPyVersion = getGraalPyVersion(getProject()); - - List includes = new ArrayList<>(getIncludes().get()); - List excludes = new ArrayList<>(getExcludes().get()); - - trim(includes); - trim(excludes); - - if (Files.isReadable(tagfile)) { - List lines; - try { - lines = Files.readAllLines(tagfile); - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to read tag file %s", tagfile), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLogger().lifecycle(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); - delete(homeDirectory); - } - if (pythonHomeChanged(includes, excludes, lines)) { - getLogger().lifecycle(String.format("Deleting GraalPy home due to changed includes or excludes")); - delete(homeDirectory); - } - } - - try { - if (!Files.exists(homeDirectory)) { - getLogger().lifecycle(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); - Files.createDirectories(homeDirectory.getParent()); - VFSUtils.copyGraalPyHome(calculateLauncherClasspath(), homeDirectory, includes, excludes, GradleLogger.of(getLogger())); - } - Files.write(tagfile, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - write(tagfile, includes, INCLUDE_PREFIX); - write(tagfile, excludes, EXCLUDE_PREFIX); - } catch (IOException | InterruptedException e) { - throw new GradleException(String.format("failed to copy graalpy home %s", homeDirectory), e); - } - } - - private void manageVenv() { - generateLaunchers(); - - Path venvDirectory = getVenvDirectory(); - - Path venvContents = venvDirectory.resolve("contents"); - List installedPackages = new ArrayList<>(); - String graalPyVersion = getGraalPyVersion(getProject()); - - var packages = getPackages().getOrElse(null); - - if (packages != null && !packages.isEmpty()) { - trim(packages); - } - - if (packages == null || packages.isEmpty()) { - getLogger().lifecycle(String.format("No venv packages declared, deleting %s", venvDirectory)); - delete(venvDirectory); - return; - } - - if (Files.isReadable(venvContents)) { - List lines = null; - try { - lines = Files.readAllLines(venvContents); - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to read tag file %s", venvContents), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLogger().lifecycle(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); - delete(venvDirectory); - } else { - for (int i = 1; i < lines.size(); i++) { - installedPackages.add(lines.get(i)); - } - } - } else { - getLogger().lifecycle(String.format("Creating GraalPy %s venv", graalPyVersion)); - } - - if (!Files.exists(venvDirectory)) { - runLauncher(getLauncherPath().toString(), "-m", "venv", venvDirectory.toString(), "--without-pip"); - runVenvBin(venvDirectory, "graalpy", "-I", "-m", "ensurepip"); - } - - deleteUnwantedPackages(venvDirectory, installedPackages, packages); - installWantedPackages(venvDirectory, installedPackages, packages); - - try { - Files.write(venvContents, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - Files.write(venvContents, packages, StandardOpenOption.APPEND); - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to write tag file %s", venvContents), e); - } - } - - private void listGraalPyResources() { - Path vfs = getHomeDirectory().getParent(); - if (Files.exists(vfs)) { - try { - VFSUtils.generateVFSFilesList(vfs); - } catch (IOException e) { - throw new GradleScriptException(String.format("Failed to generate files list in '%s'", vfs), e); - } - } - } - - private Set calculateLauncherClasspath() { - if (launcherClassPath == null) { - var addedPluginDependency = getProject().getConfigurations().getByName("runtimeClasspath").getAllDependencies().stream().filter(d -> d.getGroup().equals(GRAALPY_GROUP_ID) && d.getName().equals("python-launcher") && d.getVersion().equals(getGraalPyVersion(getProject()))).findFirst().orElseThrow(); - launcherClassPath = getProject().getConfigurations().getByName("runtimeClasspath").files(addedPluginDependency).stream().map(File::toString).collect(Collectors.toSet()); - launcherClassPath.addAll(getProject().getConfigurations().getByName("runtimeClasspath").files(getGraalPyDependency(getProject())).stream().map(File::toString).collect(Collectors.toSet())); - } - return launcherClassPath; - } - - private boolean pythonHomeChanged(List includes, List excludes, List lines) { - Set prevIncludes = new HashSet<>(); - Set prevExcludes = new HashSet<>(); - for (int i = 1; i < lines.size(); i++) { - String l = lines.get(i); - if (l.startsWith(INCLUDE_PREFIX)) { - prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); - } else if (l.startsWith(EXCLUDE_PREFIX)) { - prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); - } - } - boolean includeDidNotChange = prevIncludes.size() == includes.size() && prevIncludes.containsAll(includes); - boolean excludeDidNotChange = prevExcludes.size() == excludes.size() && prevExcludes.containsAll(excludes); - return !(includeDidNotChange && excludeDidNotChange); - } - - private void generateLaunchers() { - getLogger().lifecycle("Generating GraalPy launchers"); - var launcher = getLauncherPath(); - if (!Files.exists(launcher)) { - var java = Paths.get(System.getProperty("java.home"), "bin", "java"); - var classpath = calculateLauncherClasspath(); - if (!IS_WINDOWS) { - var script = String.format(""" - #!/usr/bin/env bash - %s -classpath %s %s --python.Executable="$0" "$@" - """, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - try { - Files.createDirectories(launcher.getParent()); - Files.writeString(launcher, script); - var perms = Files.getPosixFilePermissions(launcher); - perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); - Files.setPosixFilePermissions(launcher, perms); - } catch (IOException e) { - throw new GradleException(String.format("failed to create launcher %s", launcher), e); - } - } else { - // on windows, generate a venv launcher that executes our mvn target - var script = String.format(""" - import os, shutil, struct, venv - from pathlib import Path - vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') - tl = os.path.join(r'%s') - os.makedirs(Path(tl).parent.absolute(), exist_ok=True) - shutil.copy(vl, tl) - cmd = r'%s -classpath "%s" %s' - pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") - with open(pyvenvcfg, 'w', encoding='utf-8') as f: - f.write('venvlauncher_command = ') - f.write(cmd) - """, - launcher, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - File tmp; - try { - tmp = File.createTempFile("create_launcher", ".py"); - } catch (IOException e) { - throw new GradleScriptException("failed to create tmp launcher", e); - } - tmp.deleteOnExit(); - try (var wr = new FileWriter(tmp)) { - wr.write(script); - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to write tmp launcher %s", tmp), e); - } - runGraalPy(tmp.getAbsolutePath()); - } - } - } - - private void runGraalPy(String... args) { - var classpath = calculateLauncherClasspath(); - try { - GraalPyRunner.run(classpath, GradleLogger.of(getLogger()), args); - } catch (IOException | InterruptedException e) { - throw new GradleScriptException("failed to run Graalpy launcher", e); - } - } - - private Path getLauncherPath() { - return Paths.get(getProject().getBuildDir().getAbsolutePath(), LAUNCHER); - } - - - private static void write(Path tag, Collection list, String prefix) throws IOException { - if (list != null && !list.isEmpty()) { - Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); - } - } - - private Path getHomeDirectory() { - return getResourceDirectory(VFS_HOME); - } - - private Path getVenvDirectory() { - return getResourceDirectory(VFS_VENV); - } - - private Path getResourceDirectory(String type) { - return Path.of(getOutput().get().getAsFile().toURI()).resolve(getIncludeVfsRoot().getOrElse(true) ? VFS_ROOT : "").resolve(type); - } - - private static void delete(Path homeDirectory) { - try { - try (var s = Files.walk(homeDirectory)) { - s.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } catch (IOException e) { - throw new GradleScriptException(String.format("failed to delete %s", homeDirectory), e); - } - } - - private static void trim(List l) { - Iterator it = l.iterator(); - while (it.hasNext()) { - String p = it.next(); - if (p == null || p.trim().isEmpty()) { - it.remove(); - } - } - } - - private void installWantedPackages(Path venvDirectory, List installedPackages, List wantedPackages) { - var pkgsToInstall = new HashSet<>(wantedPackages); - installedPackages.forEach(pkgsToInstall::remove); - if (pkgsToInstall.isEmpty()) { - return; - } - getLogger().lifecycle("Installing packages {}", pkgsToInstall); - runPip(venvDirectory, "install", pkgsToInstall.toArray(new String[pkgsToInstall.size()])); - } - - private void deleteUnwantedPackages(Path venvDirectory, List installedPackages, List wantedPackages) { - List args = new ArrayList<>(installedPackages); - args.removeAll(wantedPackages); - if (args.isEmpty()) { - return; - } - args.add(0, "-y"); - getLogger().lifecycle("Removing packages {}", args); - runPip(venvDirectory, "uninstall", args.toArray(new String[args.size()])); - } - - - private void runLauncher(String launcherPath, String... args) { - try { - GraalPyRunner.runLauncher(launcherPath, GradleLogger.of(getLogger()), args); - } catch (IOException | InterruptedException e) { - throw new GradleScriptException(String.format("failed to execute launcher command %s", List.of(args)), e); - } - } - - private void runPip(Path venvDirectory, String command, String... args) { - try { - GraalPyRunner.runPip(venvDirectory, command, GradleLogger.of(getLogger()), args); - } catch (IOException | InterruptedException e) { - throw new GradleScriptException(String.format("failed to execute pip", args), e); - } - } - - private void runVenvBin(Path venvDirectory, String bin, String... args) { - try { - GraalPyRunner.runVenvBin(venvDirectory, bin, GradleLogger.of(getLogger()), args); - } catch (IOException | InterruptedException e) { - throw new GradleScriptException(String.format("failed to execute venv", args), e); - } - } -} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java similarity index 93% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java rename to graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java index 1e801af8fe..01dbe7aeac 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/GenerateManifestTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java @@ -48,7 +48,7 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -public abstract class GenerateManifestTask extends DefaultTask { +public abstract class NativeImageConfigTask extends DefaultTask { private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "graalpy-gradle-plugin"; @@ -56,11 +56,11 @@ public abstract class GenerateManifestTask extends DefaultTask { public abstract DirectoryProperty getManifestOutputDir(); @TaskAction - public void generateManifest() { + public void exec() { try { VFSUtils.writeNativeImageConfig(getManifestOutputDir().get().getAsFile().getAbsolutePath(), GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); } catch (IOException e) { - throw new GradleScriptException(e.getMessage(), e); + throw new GradleScriptException("failed to create native image configuration files", e); } } } diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java new file mode 100644 index 0000000000..e0fe26cc38 --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.tasks; + +import org.graalvm.python.GradleLogger; +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.*; +import org.gradle.api.tasks.*; +import org.gradle.api.tasks.Optional; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyDependency; +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyVersion; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.LAUNCHER_NAME; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_HOME; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_VENV; + + +public abstract class ResourcesTask extends DefaultTask { + + private Set launcherClassPath; + + @Input + @Optional + public abstract Property getIncludeVfsRoot(); + + @Input + public abstract ListProperty getPackages(); + + @Input + public abstract ListProperty getIncludes(); + + @Input + public abstract ListProperty getExcludes(); + + @OutputDirectory + public abstract DirectoryProperty getOutput(); + + @TaskAction + public void exec() { + manageHome(); + manageVenv(); + } + + private void manageHome() { + Path homeDirectory = getHomeDirectory(); + + List includes = new ArrayList<>(getIncludes().get()); + List excludes = new ArrayList<>(getExcludes().get()); + + try { + VFSUtils.createHome(homeDirectory, getGraalPyVersion(getProject()), includes, excludes, () -> calculateLauncherClasspath(), GradleLogger.of(getLogger()), (s) -> getLogger().lifecycle(s)); + } catch (IOException e) { + throw new GradleException(String.format("failed to copy graalpy home %s", homeDirectory), e); + } + } + + private void manageVenv() { + List packages = getPackages().getOrElse(null); + try { + VFSUtils.createVenv(getVenvDirectory(), new ArrayList(packages), getLauncherPath(),() -> calculateLauncherClasspath(), getGraalPyVersion(getProject()), GradleLogger.of(getLogger()), (s) -> getLogger().lifecycle(s)); + } catch (IOException e) { + throw new GradleException(String.format("failed to create venv %s", getVenvDirectory()), e); + } + } + + private Set calculateLauncherClasspath() { + if (launcherClassPath == null) { + var addedPluginDependency = getProject().getConfigurations().getByName("runtimeClasspath").getAllDependencies().stream().filter(d -> d.getGroup().equals(GRAALPY_GROUP_ID) && d.getName().equals("python-launcher") && d.getVersion().equals(getGraalPyVersion(getProject()))).findFirst().orElseThrow(); + launcherClassPath = getProject().getConfigurations().getByName("runtimeClasspath").files(addedPluginDependency).stream().map(File::toString).collect(Collectors.toSet()); + launcherClassPath.addAll(getProject().getConfigurations().getByName("runtimeClasspath").files(getGraalPyDependency(getProject())).stream().map(File::toString).collect(Collectors.toSet())); + } + return launcherClassPath; + } + + private Path getLauncherPath() { + return Paths.get(getProject().getBuildDir().getAbsolutePath(), LAUNCHER_NAME); + } + + private Path getHomeDirectory() { + return getResourceDirectory(VFS_HOME); + } + + private Path getVenvDirectory() { + return getResourceDirectory(VFS_VENV); + } + + private Path getResourceDirectory(String type) { + return Path.of(getOutput().get().getAsFile().toURI()).resolve(getIncludeVfsRoot().getOrElse(true) ? VFS_ROOT : "").resolve(type); + } + +} diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java new file mode 100644 index 0000000000..7c34a37efb --- /dev/null +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.tasks; + +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleScriptException; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; + +public abstract class VFSFilesListTask extends DefaultTask { + + @InputDirectory + public abstract Property getResourcesDir(); + + @TaskAction + public void exec() { + Path vfs = getVFSDir(); + if (Files.exists(vfs)) { + try { + VFSUtils.generateVFSFilesList(vfs); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to generate files list in '%s'", vfs), e); + } + } + } + + private Path getVFSDir() { + return Path.of(getResourcesDir().get().toURI()).resolve(VFS_ROOT); + } + +} diff --git a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java index e5c5d32b04..78e1f9ffc6 100644 --- a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java +++ b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java @@ -41,13 +41,10 @@ package org.graalvm.python.maven.plugin; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; import java.util.*; import java.util.stream.Collectors; @@ -58,26 +55,24 @@ import org.apache.maven.model.Resource; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.logging.Log; import org.apache.maven.plugins.annotations.*; import org.apache.maven.project.*; import org.eclipse.aether.graph.Dependency; -import org.graalvm.python.embedding.tools.exec.GraalPyRunner; import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.LAUNCHER_NAME; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_HOME; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_VENV; -import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; + @Mojo(name = "process-graalpy-resources", defaultPhase = LifecyclePhase.PROCESS_RESOURCES, requiresDependencyCollection = ResolutionScope.COMPILE_PLUS_RUNTIME, requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME) public class ManageResourcesMojo extends AbstractMojo { - private static final String PYTHON_LANGUAGE_ARTIFACT_ID = "python-language"; - private static final String PYTHON_RESOURCES = "python-resources"; private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; @@ -85,15 +80,6 @@ public class ManageResourcesMojo extends AbstractMojo { private static final String PYTHON_ARTIFACT_ID = "python"; private static final String GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID = "graalpy-maven-plugin"; - private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; - - private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); - private static final String LAUNCHER = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; - - private static final String INCLUDE_PREFIX = "include:"; - - private static final String EXCLUDE_PREFIX = "exclude:"; - @Parameter(defaultValue = "${project}", required = true, readonly = true) MavenProject project; @@ -150,21 +136,11 @@ public void execute() throws MojoExecutionException { } - private void trim(List l) { - Iterator it = l.iterator(); - while(it.hasNext()) { - String p = it.next(); - if(p == null || p.trim().isEmpty()) { - it.remove(); - } - } - } - private void manageNativeImageConfig() throws MojoExecutionException { try { VFSUtils.writeNativeImageConfig(project.getBuild().getOutputDirectory(), GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); } catch (IOException e) { - throw new MojoExecutionException(e); + throw new MojoExecutionException("failed to create native image configuration files", e); } } @@ -175,7 +151,7 @@ private void manageHome() throws MojoExecutionException { pythonHome.excludes = Collections.emptyList(); } else { if (pythonHome.includes != null) { - trim(pythonHome.includes); + VFSUtils.trim(pythonHome.includes); } if (pythonHome.includes == null || pythonHome.includes.isEmpty()) { pythonHome.includes = Arrays.asList(".*"); @@ -183,73 +159,26 @@ private void manageHome() throws MojoExecutionException { if (pythonHome.excludes == null) { pythonHome.excludes = Collections.emptyList(); } else { - trim(pythonHome.excludes); + VFSUtils.trim(pythonHome.excludes); } } + Path homeDirectory; if(pythonResourcesDirectory == null) { homeDirectory = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT, VFS_HOME); } else { homeDirectory = Path.of(pythonResourcesDirectory, VFS_HOME); } - var tag = homeDirectory.resolve("tagfile"); - var graalPyVersion = getGraalPyVersion(project); + List includes = toSortedArrayList(pythonHome.includes); + List excludes = toSortedArrayList(pythonHome.excludes); - List pythonHomeIncludes = toSortedArrayList(pythonHome.includes); - List pythonHomeExcludes = toSortedArrayList(pythonHome.excludes); - - if (Files.isReadable(tag)) { - List lines = null; - try { - lines = Files.readAllLines(tag); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to read tag file %s", tag), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLog().info(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); - delete(homeDirectory); - } - if (pythonHomeChanged(pythonHomeIncludes, pythonHomeExcludes, lines)) { - getLog().info(String.format("Deleting GraalPy home due to changed includes or excludes")); - delete(homeDirectory); - } - } try { - if (!Files.exists(homeDirectory)) { - getLog().info(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); - Files.createDirectories(homeDirectory.getParent()); - VFSUtils.copyGraalPyHome(calculateLauncherClasspath(project), homeDirectory, pythonHomeIncludes, pythonHomeExcludes, new MavenDelegateLog(getLog())); - } - Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - write(tag, pythonHomeIncludes, INCLUDE_PREFIX); - write(tag, pythonHomeExcludes, EXCLUDE_PREFIX); - } catch (IOException | InterruptedException e) { + VFSUtils.createHome(homeDirectory, getGraalPyVersion(project), includes, excludes, () -> calculateLauncherClasspath(project), new MavenDelegateLog(getLog()), (s) -> getLog().info(s)); + } catch(IOException e) { throw new MojoExecutionException(String.format("failed to copy graalpy home %s", homeDirectory), e); } } - private boolean pythonHomeChanged(List includes, List excludes, List lines) throws MojoExecutionException { - List prevIncludes = new ArrayList<>(); - List prevExcludes = new ArrayList<>(); - for (int i = 1; i < lines.size(); i++) { - String l = lines.get(i); - if (l.startsWith(INCLUDE_PREFIX)) { - prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); - } else if (l.startsWith(EXCLUDE_PREFIX)) { - prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); - } - } - prevIncludes = toSortedArrayList(prevIncludes); - prevExcludes = toSortedArrayList(prevExcludes); - return !(prevIncludes.equals(includes) && prevExcludes.equals(excludes)); - } - - private void write(Path tag, List list, String prefix) throws IOException { - if(list != null) { - Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); - } - } - private ArrayList toSortedArrayList(List l) { if(l != null) { Collections.sort(l); @@ -258,18 +187,6 @@ private ArrayList toSortedArrayList(List l) { return new ArrayList<>(0); } - private void delete(Path homeDirectory) throws MojoExecutionException { - try { - try (var s = Files.walk(homeDirectory)) { - s.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } catch (IOException e) { - new MojoExecutionException(String.format("failed to delete %s", homeDirectory), e); - } - } - private void listGraalPyResources() throws MojoExecutionException { Path vfs = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT); if (Files.exists(vfs)) { @@ -282,8 +199,6 @@ private void listGraalPyResources() throws MojoExecutionException { } private void manageVenv() throws MojoExecutionException { - generateLaunchers(); - Path venvDirectory; if(pythonResourcesDirectory == null) { venvDirectory = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT, VFS_VENV); @@ -291,171 +206,36 @@ private void manageVenv() throws MojoExecutionException { venvDirectory = Path.of(pythonResourcesDirectory, VFS_VENV); } - if(packages != null) { - trim(packages); - } - - if (packages == null && pythonResourcesDirectory == null) { - getLog().info(String.format("No venv packages declared, deleting %s", venvDirectory)); - delete(venvDirectory); - return; - } - - var tag = venvDirectory.resolve("contents"); - List installedPackages = new ArrayList(); - var graalPyVersion = getGraalPyVersion(project); - - if (Files.isReadable(tag)) { - List lines = null; - try { - lines = Files.readAllLines(tag); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to read tag file %s", tag), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLog().info(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); + try { + if (packages == null && pythonResourcesDirectory == null) { + getLog().info(String.format("No venv packages declared, deleting %s", venvDirectory)); delete(venvDirectory); - } else { - for (int i = 1; i < lines.size(); i++) { - installedPackages.add(lines.get(i)); - } + return; } - } else { - getLog().info(String.format("Creating GraalPy %s venv", graalPyVersion)); - } - - if (!Files.exists(venvDirectory)) { - runLauncher(getLauncherPath().toString(),"-m", "venv", venvDirectory.toString(), "--without-pip"); - runVenvBin(venvDirectory, "graalpy", "-I", "-m", "ensurepip"); - } - deleteUnwantedPackages(venvDirectory, installedPackages); - installWantedPackages(venvDirectory, installedPackages); - - try { - Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - Files.write(tag, packages, StandardOpenOption.APPEND); + VFSUtils.createVenv(venvDirectory, new ArrayList(packages), getLauncherPath(), () -> calculateLauncherClasspath(project), getGraalPyVersion(project), new MavenDelegateLog(getLog()), (s) -> getLog().info(s)); } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write tag file %s", tag), e); - } - } - - private void installWantedPackages(Path venvDirectory, List installedPackages) throws MojoExecutionException { - var pkgsToInstall = new HashSet(packages); - pkgsToInstall.removeAll(installedPackages); - if (pkgsToInstall.isEmpty()) { - return; - } - runPip(venvDirectory, "install", pkgsToInstall.toArray(new String[pkgsToInstall.size()])); - } - - private void deleteUnwantedPackages(Path venvDirectory, List installedPackages) throws MojoExecutionException { - List args = new ArrayList(installedPackages); - args.removeAll(packages); - if (args.isEmpty()) { - return; - } - args.add(0, "-y"); - runPip(venvDirectory, "uninstall", args.toArray(new String[args.size()])); - } - - private Path getLauncherPath() { - return Paths.get(project.getBuild().getDirectory(), LAUNCHER); - } - - private void generateLaunchers() throws MojoExecutionException { - getLog().info("Generating GraalPy launchers"); - var launcher = getLauncherPath(); - if (!Files.exists(launcher)) { - var java = Paths.get(System.getProperty("java.home"), "bin", "java"); - var classpath = calculateLauncherClasspath(project); - if (!IS_WINDOWS) { - var script = String.format(""" - #!/usr/bin/env bash - %s -classpath %s %s --python.Executable="$0" "$@" - """, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - try { - Files.createDirectories(launcher.getParent()); - Files.writeString(launcher, script); - var perms = Files.getPosixFilePermissions(launcher); - perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); - Files.setPosixFilePermissions(launcher, perms); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to create launcher %s", launcher), e); - } - } else { - // on windows, generate a venv launcher that executes our mvn target - var script = String.format(""" - import os, shutil, struct, venv - from pathlib import Path - vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') - tl = os.path.join(r'%s') - os.makedirs(Path(tl).parent.absolute(), exist_ok=True) - shutil.copy(vl, tl) - cmd = r'%s -classpath "%s" %s' - pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") - with open(pyvenvcfg, 'w', encoding='utf-8') as f: - f.write('venvlauncher_command = ') - f.write(cmd) - """, - launcher, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - File tmp; - try { - tmp = File.createTempFile("create_launcher", ".py"); - } catch (IOException e) { - throw new MojoExecutionException("failed to create tmp launcher", e); - } - tmp.deleteOnExit(); - try (var wr = new FileWriter(tmp)) { - wr.write(script); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write tmp launcher %s", tmp), e); - } - runGraalPy(project, getLog(), tmp.getAbsolutePath()); - } - } - } - - private void runLauncher(String launcherPath, String... args) throws MojoExecutionException { - try { - GraalPyRunner.runLauncher(launcherPath, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute launcher command %s", List.of(args))); - } - } - - private void runPip(Path venvDirectory, String command, String... args) throws MojoExecutionException { - try { - GraalPyRunner.runPip(venvDirectory, command, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute pip", args), e); + throw new MojoExecutionException(String.format("failed to create venv %s", venvDirectory), e); } } - private void runVenvBin(Path venvDirectory, String bin, String... args) throws MojoExecutionException { + private void delete(Path homeDirectory) throws MojoExecutionException { try { - GraalPyRunner.runVenvBin(venvDirectory, bin, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute venv", args), e); + try (var s = Files.walk(homeDirectory)) { + s.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (IOException e) { + new MojoExecutionException(String.format("failed to delete %s", homeDirectory), e); } } - private void runGraalPy(MavenProject project, Log log, String... args) throws MojoExecutionException { - var classpath = calculateLauncherClasspath(project); - try { - GraalPyRunner.run(classpath, new MavenDelegateLog(log), args); - } catch (IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to run Graalpy launcher"), e); - } + private Path getLauncherPath() { + return Paths.get(project.getBuild().getDirectory(), LAUNCHER_NAME); } - private static String getGraalPyVersion(MavenProject project) throws MojoExecutionException { + private static String getGraalPyVersion(MavenProject project) throws IOException { DefaultArtifact a = (DefaultArtifact) getGraalPyArtifact(project); String version = a.getVersion(); if(a.isSnapshot()) { @@ -473,13 +253,13 @@ private static String getGraalPyVersion(MavenProject project) throws MojoExecuti return version; } - private static Artifact getGraalPyArtifact(MavenProject project) throws MojoExecutionException { + private static Artifact getGraalPyArtifact(MavenProject project) throws IOException { var projectArtifacts = resolveProjectDependencies(project); Artifact graalPyArtifact = projectArtifacts.stream(). filter(a -> isPythonArtifact(a)) .findFirst() .orElse(null); - return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new MojoExecutionException("Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); + return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new IOException("Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); } private static boolean isPythonArtifact(Artifact a) { @@ -494,7 +274,7 @@ private static Collection resolveProjectDependencies(MavenProject proj .collect(Collectors.toList()); } - private Set calculateLauncherClasspath(MavenProject project) throws MojoExecutionException { + private Set calculateLauncherClasspath(MavenProject project) throws IOException { if(launcherClassPath == null) { String version = getGraalPyVersion(project); launcherClassPath = new HashSet(); @@ -504,7 +284,7 @@ private Set calculateLauncherClasspath(MavenProject project) throws Mojo getLog().debug("calculateLauncherClasspath based on " + GRAALPY_GROUP_ID + ":" + GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID + ":" + version); DefaultArtifact mvnPlugin = new DefaultArtifact(GRAALPY_GROUP_ID, GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID, version, "compile", "jar", null, new DefaultArtifactHandler("pom")); ProjectBuildingResult result = buildProjectFromArtifact(mvnPlugin); - Artifact graalPyLauncherArtifact = result.getProject().getArtifacts().stream().filter(a ->GRAALPY_GROUP_ID.equals(a.getGroupId()) && PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId())) + Artifact graalPyLauncherArtifact = result.getProject().getArtifacts().stream().filter(a -> GRAALPY_GROUP_ID.equals(a.getGroupId()) && PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId())) .findFirst() .orElse(null); // python-launcher artifact @@ -520,7 +300,7 @@ private Set calculateLauncherClasspath(MavenProject project) throws Mojo return launcherClassPath; } - private Set resolveDependencies(Artifact artifact) throws MojoExecutionException { + private Set resolveDependencies(Artifact artifact) throws IOException { Set dependencies = new HashSet<>(); ProjectBuildingResult result = buildProjectFromArtifact(artifact); for(Dependency d : result.getDependencyResolutionResult().getResolvedDependencies()) { @@ -529,7 +309,7 @@ private Set resolveDependencies(Artifact artifact) throws MojoExecutionE return dependencies; } - private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws MojoExecutionException{ + private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws IOException{ try{ ProjectBuildingRequest buildingRequest = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); buildingRequest.setProject(null); @@ -539,7 +319,7 @@ private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws return projectBuilder.build(artifact, buildingRequest); } catch (ProjectBuildingException e) { - throw new MojoExecutionException("Error while building project", e); + throw new IOException("Error while building project", e); } } diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java index df47bd3d03..f9bef8a7b3 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java @@ -61,12 +61,16 @@ public class GraalPyRunner { private static final String EXE_SUFFIX = IS_WINDOWS ? ".exe" : ""; public static void run(Set classpath, SubprocessLog log, String... args) throws IOException, InterruptedException { + run(String.join(File.pathSeparator, classpath), log, args); + } + + public static void run(String classpath, SubprocessLog log, String... args) throws IOException, InterruptedException { String workdir = System.getProperty("exec.workingdir"); Path java = Paths.get(System.getProperty("java.home"), "bin", "java"); List cmd = new ArrayList<>(); cmd.add(java.toString()); cmd.add("-classpath"); - cmd.add(String.join(File.pathSeparator, classpath)); + cmd.add(classpath); cmd.add("com.oracle.graal.python.shell.GraalPythonMain"); cmd.addAll(List.of(args)); var pb = new ProcessBuilder(cmd); diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java index d47604387b..7f093326b1 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java @@ -53,15 +53,19 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; public final class VFSUtils { @@ -84,21 +88,35 @@ public final class VFSUtils { private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; + private static final String INCLUDE_PREFIX = "include:"; + + private static final String EXCLUDE_PREFIX = "exclude:"; + + private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); + + public static final String LAUNCHER_NAME = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; + + private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; + public static void writeNativeImageConfig(String metaInfRoot, String pluginId) throws IOException { - Path metaInf = Path.of(metaInfRoot, "META-INF", "native-image", GRAALPY_GROUP_ID, pluginId); - Path resourceConfig = metaInf.resolve("resource-config.json"); + Path metaInf = Path.of(metaInfRoot, "native-image", GRAALPY_GROUP_ID, pluginId); + write(metaInf.resolve("resource-config.json"), NATIVE_IMAGE_RESOURCES_CONFIG); + write(metaInf.resolve("native-image.properties"), NATIVE_IMAGE_ARGS); + } + + private static void write(Path config, String txt) throws IOException { try { - Files.createDirectories(resourceConfig.getParent()); - Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + createParentDirectories(config); + Files.writeString(config, txt, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); } catch (IOException e) { - throw new IOException(String.format("failed to write %s", resourceConfig), e); + throw new IOException(String.format("failed to write %s", config), e); } - Path nativeImageProperties = metaInf.resolve("native-image.properties"); - try { - Files.createDirectories(nativeImageProperties.getParent()); - Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new IOException(String.format("failed to write %s", nativeImageProperties), e); + } + + private static void createParentDirectories(Path path) throws IOException { + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); } } @@ -142,6 +160,89 @@ private static String makeDirPath(Path p) { return ret; } + @FunctionalInterface + public interface LauncherClassPath { + Set get() throws IOException; + } + + public interface Log { + void info(String s); + } + + public static void createHome(Path homeDirectory, String graalPyVersion, List includes, List excludes, LauncherClassPath launcherClassPath, SubprocessLog subprocessLog, Log log) + throws IOException { + + trim(includes); + trim(excludes); + + var tag = homeDirectory.resolve("tagfile"); + + if (Files.isReadable(tag)) { + List lines = null; + try { + lines = Files.readAllLines(tag); + } catch (IOException e) { + throw new IOException(String.format("failed to read tag file %s", tag), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + log.info(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); + delete(homeDirectory); + } + if (pythonHomeChanged(includes, excludes, lines)) { + log.info(String.format("Deleting GraalPy home due to changed includes or excludes")); + delete(homeDirectory); + } + } + try { + if (!Files.exists(homeDirectory)) { + log.info(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); + createParentDirectories(homeDirectory); + VFSUtils.copyGraalPyHome(launcherClassPath.get(), homeDirectory, includes, excludes, subprocessLog); + } + Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + write(tag, includes, INCLUDE_PREFIX); + write(tag, excludes, EXCLUDE_PREFIX); + } catch (IOException | InterruptedException e) { + + throw new IOException(String.format("failed to copy graalpy home %s", homeDirectory), e); + } + } + + private static boolean pythonHomeChanged(List includes, List excludes, List lines) { + Set prevIncludes = new HashSet<>(); + Set prevExcludes = new HashSet<>(); + for (int i = 1; i < lines.size(); i++) { + String l = lines.get(i); + if (l.startsWith(INCLUDE_PREFIX)) { + prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); + } else if (l.startsWith(EXCLUDE_PREFIX)) { + prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); + } + } + boolean includeDidNotChange = prevIncludes.size() == includes.size() && prevIncludes.containsAll(includes); + boolean excludeDidNotChange = prevExcludes.size() == excludes.size() && prevExcludes.containsAll(excludes); + return !(includeDidNotChange && excludeDidNotChange); + } + + private static void write(Path tag, List list, String prefix) throws IOException { + if (list != null) { + Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); + } + } + + public static void delete(Path dir) throws IOException { + if (!Files.exists(dir)) { + return; + } + try { + try (var s = Files.walk(dir)) { + s.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } catch (IOException e) { + throw new IOException(String.format("failed to delete %s", dir), e); + } + } + public static void copyGraalPyHome(Set classpath, Path home, Collection pythonHomeIncludes, Collection pythonHomeExcludes, SubprocessLog log) throws IOException, InterruptedException { log.log(String.format("Copying std lib to '%s'\n", home)); @@ -289,4 +390,169 @@ public void log(CharSequence var1) { } } + + public static void createVenv(Path venvDirectory, List packages, Path laucherPath, LauncherClassPath launcherClassPath, String graalPyVersion, SubprocessLog subprocessLog, Log log) + throws IOException { + generateLaunchers(laucherPath, launcherClassPath, subprocessLog, log); + + if (packages != null) { + trim(packages); + } + + var tag = venvDirectory.resolve("contents"); + List installedPackages = new ArrayList<>(); + + if (Files.isReadable(tag)) { + List lines = null; + try { + lines = Files.readAllLines(tag); + } catch (IOException e) { + throw new IOException(String.format("failed to read tag file %s", tag), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + log.info(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); + delete(venvDirectory); + } else { + for (int i = 1; i < lines.size(); i++) { + installedPackages.add(lines.get(i)); + } + } + } else { + log.info(String.format("Creating GraalPy %s venv", graalPyVersion)); + } + + if (!Files.exists(venvDirectory)) { + runLauncher(laucherPath.toString(), subprocessLog, "-m", "venv", venvDirectory.toString(), "--without-pip"); + runVenvBin(venvDirectory, "graalpy", subprocessLog, "-I", "-m", "ensurepip"); + } + + if (packages != null) { + deleteUnwantedPackages(venvDirectory, packages, installedPackages, subprocessLog); + installWantedPackages(venvDirectory, packages, installedPackages, subprocessLog); + } + + try { + Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.write(tag, packages, StandardOpenOption.APPEND); + } catch (IOException e) { + throw new IOException(String.format("failed to write tag file %s", tag), e); + } + } + + private static void generateLaunchers(Path laucherPath, LauncherClassPath launcherClassPath, SubprocessLog subprocessLog, Log log) throws IOException { + if (!Files.exists(laucherPath)) { + log.info("Generating GraalPy launchers"); + createParentDirectories(laucherPath); + Path java = Paths.get(System.getProperty("java.home"), "bin", "java"); + String classpath = String.join(File.pathSeparator, launcherClassPath.get()); + if (!IS_WINDOWS) { + var script = String.format(""" + #!/usr/bin/env bash + %s -classpath %s %s --python.Executable="$0" "$@" + """, + java, + String.join(File.pathSeparator, classpath), + GRAALPY_MAIN_CLASS); + try { + Files.writeString(laucherPath, script); + var perms = Files.getPosixFilePermissions(laucherPath); + perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); + Files.setPosixFilePermissions(laucherPath, perms); + } catch (IOException e) { + throw new IOException(String.format("failed to create launcher %s", laucherPath), e); + } + } else { + // on windows, generate a venv launcher that executes our mvn target + var script = String.format(""" + import os, shutil, struct, venv + from pathlib import Path + vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') + tl = os.path.join(r'%s') + os.makedirs(Path(tl).parent.absolute(), exist_ok=True) + shutil.copy(vl, tl) + cmd = r'%s -classpath "%s" %s' + pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") + with open(pyvenvcfg, 'w', encoding='utf-8') as f: + f.write('venvlauncher_command = ') + f.write(cmd) + """, + laucherPath, + java, + classpath, + GRAALPY_MAIN_CLASS); + File tmp; + try { + tmp = File.createTempFile("create_launcher", ".py"); + } catch (IOException e) { + throw new IOException("failed to create tmp launcher", e); + } + tmp.deleteOnExit(); + try (var wr = new FileWriter(tmp)) { + wr.write(script); + } catch (IOException e) { + throw new IOException(String.format("failed to write tmp launcher %s", tmp), e); + } + + try { + GraalPyRunner.run(classpath, subprocessLog, tmp.getAbsolutePath()); + } catch (InterruptedException e) { + throw new IOException(String.format("failed to run Graalpy launcher"), e); + } + } + } + } + + private static void installWantedPackages(Path venvDirectory, List packages, List installedPackages, SubprocessLog subprocessLog) throws IOException { + Set pkgsToInstall = new HashSet<>(packages); + pkgsToInstall.removeAll(installedPackages); + if (pkgsToInstall.isEmpty()) { + return; + } + runPip(venvDirectory, "install", subprocessLog, pkgsToInstall.toArray(new String[pkgsToInstall.size()])); + } + + private static void deleteUnwantedPackages(Path venvDirectory, List packages, List installedPackages, SubprocessLog subprocessLog) throws IOException { + List args = new ArrayList<>(installedPackages); + args.removeAll(packages); + if (args.isEmpty()) { + return; + } + args.add(0, "-y"); + runPip(venvDirectory, "uninstall", subprocessLog, args.toArray(new String[args.size()])); + } + + private static void runLauncher(String launcherPath, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runLauncher(launcherPath, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute launcher command %s", List.of(args))); + } + } + + private static void runPip(Path venvDirectory, String command, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runPip(venvDirectory, command, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute pip %s", List.of(args)), e); + } + } + + private static void runVenvBin(Path venvDirectory, String bin, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runVenvBin(venvDirectory, bin, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute venv %s", List.of(args)), e); + } + } + + public static List trim(List l) { + Iterator it = l.iterator(); + while (it.hasNext()) { + String p = it.next(); + if (p == null || p.trim().isEmpty()) { + it.remove(); + } + } + return l; + } } From 2e447fdeca1e01e9449753f7149d96ee6095ca66 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Thu, 18 Jul 2024 16:58:05 +0200 Subject: [PATCH 05/16] skip getIncludeVfsRootDir from GraalPy extension --- .../graalvm/python/GraalPyGradlePlugin.java | 39 ++++++++++--------- .../graalvm/python/dsl/GraalPyExtension.java | 13 ++++++- .../graalvm/python/dsl/PythonHomeInfo.java | 3 ++ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index 3112db0f82..71f5810294 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -58,6 +58,7 @@ import java.util.List; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; public abstract class GraalPyGradlePlugin implements Plugin { @@ -67,7 +68,8 @@ public abstract class GraalPyGradlePlugin implements Plugin { private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; private static final String PYTHON_ARTIFACT_ID = "python"; private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalPy"; - private static final String DEFAULT_WRAPPER_DIRECTORY = "python-generated"; + private static final String DEFAULT_RESOURCES_DIRECTORY = "graalpy-resources"; + private static final String GRAALPY_META_INF_DIRECTORY = "graalpy-meta-inf"; private static final String GRAALPY_RESOURCES_TASK = "graalPyResources"; private static final String GRAALPY_NATIVE_IMAGE_CONFIG_TASK = "graalPyNativeImageConfig"; private static final String GRAALPY_VFS_FILESLIST_TASK = "graalPyVFSFilesList"; @@ -86,27 +88,32 @@ public void apply(Project project) { extension.getPythonHome().getExcludes().convention(Collections.emptyList()); extension.getPackages().convention(Collections.emptyList()); - TaskProvider installPackagesTask = project.getTasks().register(GRAALPY_RESOURCES_TASK, ResourcesTask.class); - installPackagesTask.configure(t -> { + TaskProvider resourcesTask = project.getTasks().register(GRAALPY_RESOURCES_TASK, ResourcesTask.class); + resourcesTask.configure(t -> { t.getIncludes().set(extension.getPythonHome().getIncludes()); t.getExcludes().set(extension.getPythonHome().getExcludes()); t.getPackages().set(extension.getPackages()); - t.getIncludeVfsRoot().set(extension.getIncludeVfsRootDir()); - t.getOutput().set(extension.getPythonResourcesDirectory()); + if(extension.getPythonResourcesDirectory().isPresent()) { + t.getOutput().set(extension.getPythonResourcesDirectory()); + t.getIncludeVfsRoot().set(false); + } else { + t.getOutput().set(project.getLayout().getBuildDirectory().dir(DEFAULT_RESOURCES_DIRECTORY)); + t.getIncludeVfsRoot().set(true); + } t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); - TaskProvider generateManifestTask = project.getTasks().register(GRAALPY_NATIVE_IMAGE_CONFIG_TASK, NativeImageConfigTask.class); - project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(generateManifestTask)); - generateManifestTask.configure(t -> { - t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir("GRAAL-META-INF")); + TaskProvider nativeImageConfigTask = project.getTasks().register(GRAALPY_NATIVE_IMAGE_CONFIG_TASK, NativeImageConfigTask.class); + project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(nativeImageConfigTask)); + nativeImageConfigTask.configure(t -> { + t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir(GRAALPY_META_INF_DIRECTORY)); t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); - TaskProvider generateVFSFilesListTask = project.getTasks().register(GRAALPY_VFS_FILESLIST_TASK, VFSFilesListTask.class); - generateVFSFilesListTask.configure(t -> { + TaskProvider vfsFilesListTask = project.getTasks().register(GRAALPY_VFS_FILESLIST_TASK, VFSFilesListTask.class); + vfsFilesListTask.configure(t -> { t.getResourcesDir().convention((((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).getDestinationDir())); t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); @@ -114,15 +121,12 @@ public void apply(Project project) { project.afterEvaluate(p -> { checkAndAddDependencies(); - if (!extension.getPythonResourcesDirectory().isPresent()) { - ((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).with(project.copySpec().from(installPackagesTask)); + ((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).with(project.copySpec().from(resourcesTask)); + } else { + project.getTasks().getByName(JavaPlugin.CLASSES_TASK_NAME, t -> t.dependsOn(GRAALPY_RESOURCES_TASK)); } - - // Provide the default value after the isPresent check, otherwise isPresent always returns true - extension.getPythonResourcesDirectory().convention(project.getLayout().getBuildDirectory().dir(DEFAULT_WRAPPER_DIRECTORY)); }); - } private void checkAndAddDependencies() { @@ -130,7 +134,6 @@ private void checkAndAddDependencies() { project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_EMBEDDING_ARTIFACT_ID, getGraalPyVersion(project))); } - public static String getGraalPyVersion(Project project) { return getGraalPyDependency(project).getVersion(); } diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java index 6dec18ad75..8b5af37b72 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java @@ -47,12 +47,21 @@ import org.gradle.api.tasks.Nested; public interface GraalPyExtension { + + /** + * External directory supposed to be populated with python resources, namely graalpy stdlib and venv. + * It is either present or not. + */ DirectoryProperty getPythonResourcesDirectory(); + /** + * Determines third party python packages to be installed for graalpy usage. + */ ListProperty getPackages(); - Property getIncludeVfsRootDir(); - + /** + * Determines what parts of graalpy stdlib are supposed to be available for graalpy. + */ @Nested PythonHomeInfo getPythonHome(); diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java index bd574c2bca..18f4ddbf36 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java @@ -42,6 +42,9 @@ import org.gradle.api.provider.SetProperty; +/** + * Determines what parts of graalpy stdlib are supposed to be avalable for graalpy. + */ public interface PythonHomeInfo { SetProperty getIncludes(); SetProperty getExcludes(); From c6f71cf2f4fb7fe9cf84e96c51238add301d15a8 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Thu, 18 Jul 2024 17:34:56 +0200 Subject: [PATCH 06/16] generate graalpy resources and meta-inf into build/generated/graalpy --- .../org/graalvm/python/GraalPyGradlePlugin.java | 17 ++++++++--------- ...iveImageConfigTask.java => MetaInfTask.java} | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) rename graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/{NativeImageConfigTask.java => MetaInfTask.java} (97%) diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index 71f5810294..74a0f187ec 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -42,7 +42,7 @@ import org.graalvm.python.dsl.GraalPyExtension; import org.graalvm.python.tasks.VFSFilesListTask; -import org.graalvm.python.tasks.NativeImageConfigTask; +import org.graalvm.python.tasks.MetaInfTask; import org.graalvm.python.tasks.ResourcesTask; import org.gradle.api.GradleException; import org.gradle.api.Plugin; @@ -54,12 +54,11 @@ import org.gradle.jvm.tasks.Jar; import org.gradle.language.jvm.tasks.ProcessResources; +import java.io.File; import java.util.Collections; import java.util.List; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; -import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; - public abstract class GraalPyGradlePlugin implements Plugin { private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; @@ -68,10 +67,10 @@ public abstract class GraalPyGradlePlugin implements Plugin { private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; private static final String PYTHON_ARTIFACT_ID = "python"; private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalPy"; - private static final String DEFAULT_RESOURCES_DIRECTORY = "graalpy-resources"; - private static final String GRAALPY_META_INF_DIRECTORY = "graalpy-meta-inf"; + private static final String DEFAULT_RESOURCES_DIRECTORY = "generated" + File.separator + "graalpy" + File.separator + "resources"; + private static final String GRAALPY_META_INF_DIRECTORY = "generated" + File.separator + "graalpy" + File.separator + "META-INF"; private static final String GRAALPY_RESOURCES_TASK = "graalPyResources"; - private static final String GRAALPY_NATIVE_IMAGE_CONFIG_TASK = "graalPyNativeImageConfig"; + private static final String GRAALPY_META_INF_TASK_TASK = "graalPyMetaInf"; private static final String GRAALPY_VFS_FILESLIST_TASK = "graalPyVFSFilesList"; @@ -105,9 +104,9 @@ public void apply(Project project) { t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); - TaskProvider nativeImageConfigTask = project.getTasks().register(GRAALPY_NATIVE_IMAGE_CONFIG_TASK, NativeImageConfigTask.class); - project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(nativeImageConfigTask)); - nativeImageConfigTask.configure(t -> { + TaskProvider metaInfTask = project.getTasks().register(GRAALPY_META_INF_TASK_TASK, MetaInfTask.class); + project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(metaInfTask)); + metaInfTask.configure(t -> { t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir(GRAALPY_META_INF_DIRECTORY)); t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); }); diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java similarity index 97% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java rename to graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java index 01dbe7aeac..afa389a25a 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/NativeImageConfigTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java @@ -48,7 +48,7 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -public abstract class NativeImageConfigTask extends DefaultTask { +public abstract class MetaInfTask extends DefaultTask { private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "graalpy-gradle-plugin"; From 89e338dc3a32202c58d96d0f6d6025e4658a4ca7 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Thu, 25 Jul 2024 12:06:17 +0200 Subject: [PATCH 07/16] set default for empty includes --- .../main/java/org/graalvm/python/GraalPyGradlePlugin.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java index 74a0f187ec..9854f847e2 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -89,7 +89,12 @@ public void apply(Project project) { TaskProvider resourcesTask = project.getTasks().register(GRAALPY_RESOURCES_TASK, ResourcesTask.class); resourcesTask.configure(t -> { - t.getIncludes().set(extension.getPythonHome().getIncludes()); + if(extension.getPythonHome().getIncludes().get().isEmpty()) { + t.getIncludes().set(List.of(".*")); + } else { + t.getIncludes().set(extension.getPythonHome().getIncludes()); + } + t.getExcludes().set(extension.getPythonHome().getExcludes()); t.getPackages().set(extension.getPackages()); From 2b9d94f26b1faddeaa4eecc865abeba86d3b2638 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Mon, 29 Jul 2024 22:12:12 +0200 Subject: [PATCH 08/16] provide whole META-INF path from ManageResourcesMojo --- .../main/java/org/graalvm/python/tasks/MetaInfTask.java | 2 +- .../graalvm/python/maven/plugin/ManageResourcesMojo.java | 2 +- .../org/graalvm/python/embedding/tools/vfs/VFSUtils.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java index afa389a25a..5e5a7762eb 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java +++ b/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java @@ -58,7 +58,7 @@ public abstract class MetaInfTask extends DefaultTask { @TaskAction public void exec() { try { - VFSUtils.writeNativeImageConfig(getManifestOutputDir().get().getAsFile().getAbsolutePath(), GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); + VFSUtils.writeNativeImageConfig(getManifestOutputDir().get().getAsFile().toPath(), GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); } catch (IOException e) { throw new GradleScriptException("failed to create native image configuration files", e); } diff --git a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java index 78e1f9ffc6..8150d8b4e3 100644 --- a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java +++ b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java @@ -138,7 +138,7 @@ public void execute() throws MojoExecutionException { private void manageNativeImageConfig() throws MojoExecutionException { try { - VFSUtils.writeNativeImageConfig(project.getBuild().getOutputDirectory(), GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); + VFSUtils.writeNativeImageConfig(Path.of(project.getBuild().getOutputDirectory(), "META-INF"), GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); } catch (IOException e) { throw new MojoExecutionException("failed to create native image configuration files", e); } diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java index 7f093326b1..b4b95dbf11 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java @@ -98,10 +98,10 @@ public final class VFSUtils { private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; - public static void writeNativeImageConfig(String metaInfRoot, String pluginId) throws IOException { - Path metaInf = Path.of(metaInfRoot, "native-image", GRAALPY_GROUP_ID, pluginId); - write(metaInf.resolve("resource-config.json"), NATIVE_IMAGE_RESOURCES_CONFIG); - write(metaInf.resolve("native-image.properties"), NATIVE_IMAGE_ARGS); + public static void writeNativeImageConfig(Path metaInfRoot, String pluginId) throws IOException { + Path p = metaInfRoot.resolve(Path.of("native-image", GRAALPY_GROUP_ID, pluginId)); + write(p.resolve("resource-config.json"), NATIVE_IMAGE_RESOURCES_CONFIG); + write(p.resolve("native-image.properties"), NATIVE_IMAGE_ARGS); } private static void write(Path config, String txt) throws IOException { From 6023deb2520e6284a4d1401a31039b521453681c Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Mon, 29 Jul 2024 22:12:44 +0200 Subject: [PATCH 09/16] gradle plugin tests --- .../standalone/gradle/build/build.gradle | 21 + .../standalone/gradle/build/build.gradle.kts | 26 ++ .../standalone/gradle/build/settings.gradle | 12 + .../gradle/build/settings.gradle.kts | 10 + .../src/main/java/org/example/GraalPy.java | 80 ++++ .../src/main/java/org/example/Hello.java | 45 ++ .../META-INF/native-image/proxy-config.json | 3 + .../org.graalvm.python.vfs/src/hello.py | 48 ++ .../src/tests/standalone/test_standalone.py | 426 +++++++++++++++++- .../src/tests/standalone/util.py | 17 +- mx.graalpython/mx_graalpython.py | 2 + 11 files changed, 681 insertions(+), 9 deletions(-) create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.java create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.java create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle new file mode 100644 index 0000000000..b03dcb7441 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle @@ -0,0 +1,21 @@ +plugins { + id "application" + id "org.graalvm.python" + id "org.graalvm.buildtools.native" version "0.10.2" +} + +repositories { + mavenCentral() +} + +application { + mainClass = "org.example.GraalPy" +} + +dependencies { + implementation("org.graalvm.python:python-community:24.2.0") +} + +run { + enableAssertions = true +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts new file mode 100644 index 0000000000..626877d7a7 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + application + id("org.graalvm.python") + id("org.graalvm.buildtools.native") version "0.10.2" +} + +repositories { + mavenCentral() +} + +application { + // Define the main class for the application. + mainClass = "org.example.GraalPy" +} + +// XXX how to run with asserts on, aka -ea? +//run { +// enableAssertions = true +//} +val r = tasks.run.get() +r.enableAssertions = true +r.outputs.upToDateWhen {false} + +dependencies { + implementation("org.graalvm.python:python-community:24.2.0") +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle new file mode 100644 index 0000000000..94a40b69a8 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle @@ -0,0 +1,12 @@ +rootProject.name = "graalpy-gradle-test-project" + +buildscript { + repositories { + maven { + url "https://repo.gradle.org/gradle/libs-releases/" + } + } + dependencies { + classpath "org.graalvm.python:graalpy-gradle-plugin:24.2.0" + } +} \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts new file mode 100644 index 0000000000..3353bd017a --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts @@ -0,0 +1,10 @@ +rootProject.name = "graalpy-gradle-test-project" + +buildscript { + repositories { + maven(url="https://repo.gradle.org/gradle/libs-releases/") + } + dependencies { + classpath("org.graalvm.python:graalpy-gradle-plugin:24.2.0") + } +} \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.java b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.java new file mode 100644 index 0000000000..6b778e2804 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.example; + +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; +import java.io.IOException; + +import org.graalvm.python.embedding.utils.GraalPyResources; + +public class GraalPy { + private static final String PYTHON = "python"; + + public static void main(String[] args) { + try (Context context = GraalPyResources.createContext()) { + Source source; + try { + source = Source.newBuilder(PYTHON, "import hello", "").internal(true).build(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + context.eval(source); + + // retrieve the python PyHello class + Value pyHelloClass = context.getPolyglotBindings().getMember("PyHello"); + Value pyHello = pyHelloClass.newInstance(); + // and cast it to the Hello interface which matches PyHello + Hello hello = pyHello.as(Hello.class); + hello.hello("java"); + + } catch (PolyglotException e) { + if (e.isExit()) { + System.exit(e.getExitStatus()); + } else { + throw e; + } + } + } +} \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.java b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.java new file mode 100644 index 0000000000..61126a408f --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.example; + +public interface Hello { + void hello(String txt); +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json new file mode 100644 index 0000000000..0edc8f9ac2 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json @@ -0,0 +1,3 @@ +[ + ["org.example.Hello"] +] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py new file mode 100644 index 0000000000..cd39ab8cb2 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import polyglot +from termcolor import colored + +class PyHello: + def hello(self, txt): + colored_text = colored("hello " + str(txt), "red", attrs=["reverse", "blink"]) + print(colored_text) + +# We export the PyHello class to Java as our explicit interface with the Java side +polyglot.export_value("PyHello", PyHello) \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py index c8dd9bb268..234e73e4fb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py @@ -44,12 +44,22 @@ import urllib.parse import shutil import util +from abc import ABC, abstractmethod is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true" +is_gradle_enabled = 'ENABLE_GRADLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_GRADLE_STANDALONE_UNITTESTS'] == "true" GLOBAL_MVN_CMD = [shutil.which('mvn'), "--batch-mode"] VFS_PREFIX = "org.graalvm.python.vfs" +GRAALPY_EMPTY = "graalPy {}" + +GRAALPY_EMPTY_HOME = """ +graalPy { + pythonHome { } +} +""" + def get_gp(): graalpy = util.get_gp() @@ -78,13 +88,13 @@ def get_gp(): def replace_in_file(file, str, replace_str): with open(file, "r") as f: contents = f.read() + assert str in contents with open(file, "w") as f: f.write(contents.replace(str, replace_str)) -class PolyglotAppTest(unittest.TestCase): - +class PolyglotAppTestBase(unittest.TestCase): def setUpClass(self): - if not is_enabled: + if not is_enabled and not is_gradle_enabled: return self.env = os.environ.copy() @@ -155,6 +165,409 @@ def setUpClass(self): assert return_code == 0 break +def append(file, txt): + with open(file, "a") as f: + f.write(txt) + +class PolyglotAppGradleTestBase(PolyglotAppTestBase, ABC): + + @abstractmethod + def target_dir_name_sufix(self, target_dir): + pass + + @abstractmethod + def copy_build_files(self, target_dir): + pass + + @abstractmethod + def packages_termcolor(self, build_file): + pass + + @abstractmethod + def packages_termcolor_ujson(self): + pass + + @abstractmethod + def packages_termcolor_resource_dir(self, resources_dir): + pass + + @abstractmethod + def empty_home_includes(self): + pass + + @abstractmethod + def home_includes(self): + pass + + @abstractmethod + def empty_packages(self): + pass + + def generate_app(self, tmpdir, target_dir): + shutil.copytree(os.path.join(os.path.dirname(__file__), "gradle", "gradle-test-project"), target_dir) + self.copy_build_files(target_dir) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_generated_gradle_app(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "generated_app_gradle" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + append(build_file, self.packages_termcolor()) + + gradlew_cmd = util.get_gradle_wrapper(target_dir, self.env) + + # build + cmd = gradlew_cmd + ["nativeCompile"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + + # check fileslist.txt + fl_path = os.path.join(target_dir, "build", "resources", "main", VFS_PREFIX, "fileslist.txt") + with open(fl_path) as f: + lines = f.readlines() + assert "/" + VFS_PREFIX + "/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/lib-graalpython/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/lib-python/\n" in lines, "unexpected output from " + str(cmd) + + # execute and check native image + cmd = [os.path.join(target_dir, "build", "native", "nativeCompile", "graalpy-gradle-test-project")] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("hello java", out) + + # import struct from python file triggers extract of native extension files in VirtualFileSystem + hello_src = os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py") + contents = open(hello_src, 'r').read() + with open(hello_src, 'w') as f: + f.write("import struct\n" + contents) + + # rebuild and exec + cmd = gradlew_cmd + ["build", "run"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("hello java", out) + + #GR-51132 - NoClassDefFoundError when running polyglot app in java mode + util.check_ouput("java.lang.NoClassDefFoundError", out, False) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_generated_gradle_app_external_resources(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "generated_gradle_app_external_resources" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + + # patch project to use external directory for resources + resources_dir = os.path.join(target_dir, "python-resources") + os.makedirs(resources_dir, exist_ok=True) + src_dir = os.path.join(resources_dir, "src") + os.makedirs(src_dir, exist_ok=True) + # copy hello.py + shutil.copyfile(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py"), os.path.join(src_dir, "hello.py")) + shutil.rmtree(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs")) + # patch GraalPy.java + replace_in_file(os.path.join(target_dir, "src", "main", "java", "org", "example", "GraalPy.java"), + "package org.example;", + "package org.example;\nimport java.nio.file.Path;") + replace_in_file(os.path.join(target_dir, "src", "main", "java", "org", "example", "GraalPy.java"), + "GraalPyResources.createContext()", + "GraalPyResources.contextBuilder(Path.of(\"" + resources_dir + "\")).build()") + + # patch build.gradle + append(build_file, self.packages_termcolor_resource_dir(resources_dir)) + + # build + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + cmd = gradle_cmd + ["clean", "nativeCompile"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + + # execute and check native image + cmd = [os.path.join(target_dir, "build", "native", "nativeCompile", "graalpy-gradle-test-project")] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("hello java", out) + + # 2.) check java build and exec + cmd = gradle_cmd + ["run"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("hello java", out) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_fail_without_graalpy_dep(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "gradle_fail_without_graalpy_dep" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + append(build_file, GRAALPY_EMPTY) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + replace_in_file(build_file, + "implementation(\"org.graalvm.python:python-community:24.2.0\")", + "// implementation(\"org.graalvm.python:python-community:24.2.0\")") + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("Missing GraalPy dependency. Please add to your build.gradle either org.graalvm.polyglot:python-community or org.graalvm.polyglot:python", out) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_gen_launcher_and_venv(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "gradle_gen_launcher_and_venv" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + + build_file = os.path.join(target_dir, self.build_file_name) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + append(build_file, self.packages_termcolor_ujson()) + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("-m venv", out) + util.check_ouput("-m ensurepip",out) + util.check_ouput("ujson", out) + util.check_ouput("termcolor", out) + + # run again and assert that we do not regenerate the venv + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("-m venv", out, False) + util.check_ouput("-m ensurepip", out, False) + util.check_ouput("ujson", out, False) + util.check_ouput("termcolor", out, False) + + # remove ujson pkg from plugin config and check if unistalled + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) + append(build_file, self.packages_termcolor()) + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("-m venv", out, False) + util.check_ouput("-m ensurepip", out, False) + util.check_ouput("Uninstalling ujson", out) + util.check_ouput("termcolor", out, False) + + def check_tagfile(self, home, expected): + with open(os.path.join(home, "tagfile")) as f: + lines = f.readlines() + assert lines == expected, "expected tagfile " + str(expected) + ", but got " + str(lines) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_check_home(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "check_home_test" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + + build_file_template = os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name) + build_file = os.path.join(target_dir, self.build_file_name) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + process_resources_cmd = gradle_cmd + ["graalPyResources"] + + # 1. process-resources with no pythonHome config + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out) + + home_dir = os.path.join(target_dir, "build", "generated", "graalpy", "resources", VFS_PREFIX, "home") + assert os.path.exists(home_dir) + assert os.path.exists(os.path.join(home_dir, "lib-graalpython")) + assert os.path.isdir(os.path.join(home_dir, "lib-graalpython")) + assert os.path.exists(os.path.join(home_dir, "lib-python")) + assert os.path.isdir(os.path.join(home_dir, "lib-python")) + assert os.path.exists(os.path.join(home_dir, "tagfile")) + assert os.path.isfile(os.path.join(home_dir, "tagfile")) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 2. process-resources with empty pythonHome + shutil.copyfile(build_file_template, build_file) + append(build_file, GRAALPY_EMPTY_HOME) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out, False) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 3. process-resources with empty pythonHome includes and excludes + shutil.copyfile(build_file_template, build_file) + append(build_file, self.empty_home_includes()) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out, False) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 4. process-resources with pythonHome includes and excludes + shutil.copyfile(build_file_template, build_file) + append(build_file, self.home_includes()) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Deleting GraalPy home due to changed includes or excludes", out) + util.check_ouput("Copying std lib to ", out) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*__init__.py\n', 'exclude:.*html/__init__.py\n']) + + # 5. check fileslist.txt + # XXX build vs graalPyVFSFilesList task? + out, return_code = util.run_cmd(gradle_cmd + ["build"], self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + fl_path = os.path.join(target_dir, "build", "resources", "main", VFS_PREFIX, "fileslist.txt") + with open(fl_path) as f: + for line in f: + line = f.readline() + # string \n + line = line[:len(line)-1] + if line.endswith("tagfile"): + continue + if not line.startswith("/" + VFS_PREFIX + "/home/") or line.endswith("/"): + continue + assert line.endswith("/__init__.py"), f"expected line to end with /__init__.py, but was '{line}'" + assert not line.endswith("html/__init__.py"), f"expected line to end with html/__init__.py, but was '{line}''" + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_empty_packages(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "empty_packages_test" + self.target_dir_name_sufix()) + self.generate_app(tmpdir, target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + + append(build_file, self.empty_packages()) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + +class PolyglotAppGradleGroovyTest(PolyglotAppGradleTestBase): + + def setUpClass(self): + super().setUpClass() + self.build_file_name = "build.gradle" + self.settings_file_name = "settings.gradle" + + def target_dir_name_sufix(self): + return "_groovy" + + def copy_build_files(self, target_dir): + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), os.path.join(target_dir, self.settings_file_name)) + + def packages_termcolor(self): + return """ +graalPy { + packages = ["termcolor"] +} +""" + + def packages_termcolor_ujson(self): + return """ +graalPy { + packages = ["termcolor", "ujson"] +} +""" + + def packages_termcolor_resource_dir(self, resources_dir): + return f""" +graalPy {{ + packages = ["termcolor"] + pythonResourcesDirectory = file("{resources_dir}") +}} +""" + + def home_includes(self): + return """ +graalPy { + pythonHome { + includes = [".*__init__.py"] + excludes = [".*html/__init__.py"] + } +} +""" + + def empty_home_includes(self): + return """ +graalPy { + pythonHome { + includes = [] + excludes = [] + } +} +""" + + def empty_packages(self): + return """ +graalPy { + packages = [] +} +""" + +class PolyglotAppGradleKotlinTest(PolyglotAppGradleTestBase): + + def setUpClass(self): + super().setUpClass() + self.build_file_name = "build.gradle.kts" + self.settings_file_name = "settings.gradle.kts" + + def target_dir_name_sufix(self): + return "_kotlin" + + def copy_build_files(self, target_dir): + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), os.path.join(target_dir, self.settings_file_name)) + + def packages_termcolor(self): + return """ +graalPy { + packages.add("termcolor") +} +""" + + def packages_termcolor_ujson(self): + return """ +graalPy { + packages.add("termcolor") + packages.add("ujson") +} +""" + + def packages_termcolor_resource_dir(self, resources_dir): + return f""" +graalPy {{ + packages.add("termcolor") + pythonResourcesDirectory = file("{resources_dir}") +}} +""" + + def home_includes(self): + return """ +graalPy { + pythonHome { + includes.add(".*__init__.py") + excludes.add(".*html/__init__.py") + } +} +""" + + def empty_home_includes(self): + return """ +graalPy { + pythonHome { + includes + excludes + } +} +""" + + def empty_packages(self): + return """ +graalPy { + packages +} +""" + +class PolyglotAppTest(PolyglotAppTestBase): + def generate_app(self, tmpdir, target_dir, target_name, pom_template=None): cmd = GLOBAL_MVN_CMD + [ "archetype:generate", @@ -253,7 +666,7 @@ def test_generated_app_external_resources(self): self.generate_app(tmpdir, target_dir, target_name) # patch project to use external directory for resources - resources_dir = os.path.join(target_dir, "python") + resources_dir = os.path.join(target_dir, "python-resources") os.makedirs(resources_dir, exist_ok=True) src_dir = os.path.join(resources_dir, "src") os.makedirs(src_dir, exist_ok=True) @@ -266,12 +679,12 @@ def test_generated_app_external_resources(self): "package it.pkg;\nimport java.nio.file.Path;") replace_in_file(os.path.join(target_dir, "src", "main", "java", "it", "pkg", "GraalPy.java"), "GraalPyResources.createContext()", - "GraalPyResources.contextBuilder(Path.of(\"python\")).build()") + "GraalPyResources.contextBuilder(Path.of(\"python-resources\")).build()") # patch pom.xml replace_in_file(os.path.join(target_dir, "pom.xml"), "", - "${project.basedir}/python\n") + "${project.basedir}/python-resources\n") mvnw_cmd = util.get_mvn_wrapper(target_dir, self.env) @@ -298,7 +711,6 @@ def test_generated_app_external_resources(self): util.check_ouput("BUILD SUCCESS", out) util.check_ouput("hello java", out) - @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true") def test_fail_without_graalpy_dep(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py index 0a44ef5b84..9254c71b79 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py @@ -43,6 +43,7 @@ import sys MAVEN_VERSION = "3.9.8" +GRADLE_VERSION = "8.8" def run_cmd(cmd, env, cwd=None, print_out=False): out = [] @@ -66,12 +67,13 @@ def check_ouput(txt, out, contains=True): print_output(out, f"did not expect '{txt}' in output") assert False -def print_output(out, err_msg): +def print_output(out, err_msg=None): print("============== output =============") for line in out: print(line, end="") print("\n========== end of output ==========") - print("", err_msg, "", sep="\n") + if err_msg: + print("", err_msg, "", sep="\n") def get_mvn_wrapper(project_dir, env): cmd = "mvnw" if 'win32' != sys.platform else "mvnw.cmd" @@ -81,6 +83,17 @@ def get_mvn_wrapper(project_dir, env): check_ouput(MAVEN_VERSION, out) return mvn_cmd +def get_gradle_wrapper(project_dir, env): + gradle = shutil.which('gradle') + cmd = [gradle, "wrapper", f"--gradle-version={GRADLE_VERSION}"] + out, return_code = run_cmd(cmd, env, cwd=project_dir) + check_ouput("BUILD SUCCESS", out) + gradle_cmd = [os.path.abspath(os.path.join(project_dir, "gradlew" if 'win32' != sys.platform else "gradlew.bat"))] + cmd = gradle_cmd + ["--version"] + out, return_code = run_cmd(cmd, env, cwd=project_dir) + check_ouput(GRADLE_VERSION, out) + return gradle_cmd + def get_gp(): if "PYTHON_STANDALONE_HOME" not in os.environ: print_missing_graalpy_msg() diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index c8915b9959..847dc1556c 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1537,6 +1537,8 @@ def graalpython_gate_runner(args, tasks): mvn_repo_path, version, env = deploy_local_maven_repo() env['ENABLE_STANDALONE_UNITTESTS'] = 'true' + # TODO disabled until gradle available on gate + # env['ENABLE_ENABLE_GRADLE_STANDALONE_UNITTESTS'] = 'true' env['ENABLE_JBANG_INTEGRATION_UNITTESTS'] ='true' env['JAVA_HOME'] = gvm_jdk env['PYTHON_STANDALONE_HOME'] = standalone_home From d32eb2249f8b9d1b4e29a7f23e3c647772141b08 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Tue, 30 Jul 2024 13:28:01 +0200 Subject: [PATCH 10/16] fixed licenses date in graalpy-gradle-plugin --- graalpython/graalpy-gradle-plugin/pom.xml | 2 +- .../src/main/java/org/graalvm/python/GraalPyGradlePlugin.java | 4 +--- .../src/main/java/org/graalvm/python/GradleLogger.java | 2 +- .../main/java/org/graalvm/python/dsl/GraalPyExtension.java | 2 +- .../src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java | 2 +- .../src/main/java/org/graalvm/python/tasks/MetaInfTask.java | 2 +- .../src/main/java/org/graalvm/python/tasks/ResourcesTask.java | 2 +- .../main/java/org/graalvm/python/tasks/VFSFilesListTask.java | 2 +- 8 files changed, 8 insertions(+), 10 deletions(-) diff --git a/graalpython/graalpy-gradle-plugin/pom.xml b/graalpython/graalpy-gradle-plugin/pom.xml index e5daa9a08a..4e50ced5c2 100644 --- a/graalpython/graalpy-gradle-plugin/pom.xml +++ b/graalpython/graalpy-gradle-plugin/pom.xml @@ -1,6 +1,6 @@ Dly_^lOSy&zIIhm*HXm1?VS=_iacG);_I9c zUQH1>i#*?oPIwBMJkzi_*>HoUe}_4o>2(SHWzqQ=;TyhAHS;Enr7!#8;sdlty&(>d zl%5cjri8`2X^Ds`jnw7>A`X|bl=U8n+3LKLy(1dAu8`g@9=5iw$R0qk)w8Vh_Dt^U zIglK}sn^)W7aB(Q>HvrX=rxB z+*L)3DiqpQ_%~|m=44LcD4-bxO3OO*LPjsh%p(k?&jvLp0py57oMH|*IMa(<|{m1(0S|x)?R-mqJ=I;_YUZA>J z62v*eSK;5w!h8J+6Z2~oyGdZ68waWfy09?4fU&m7%u~zi?YPHPgK6LDwphgaYu%0j zurtw)AYOpYKgHBrkX189mlJ`q)w-f|6>IER{5Lk97%P~a-JyCRFjejW@L>n4vt6#hq;!|m;hNE||LK3nw1{bJOy+eBJjK=QqNjI;Q6;Rp5 z&035pZDUZ#%Oa;&_7x0T<7!RW`#YBOj}F380Bq?MjjEhrvlCATPdkCTTl+2efTX$k zH&0zR1n^`C3ef~^sXzJK-)52(T}uTG%OF8yDhT76L~|^+hZ2hiSM*QA9*D5odI1>& z9kV9jC~twA5MwyOx(lsGD_ggYmztXPD`2=_V|ks_FOx!_J8!zM zTzh^cc+=VNZ&(OdN=y4Juw)@8-85lwf_#VMN!Ed(eQiRiLB2^2e`4dp286h@v@`O%_b)Y~A; zv}r6U?zs&@uD_+(_4bwoy7*uozNvp?bXFoB8?l8yG0qsm1JYzIvB_OH4_2G*IIOwT zVl%HX1562vLVcxM_RG*~w_`FbIc!(T=3>r528#%mwwMK}uEhJ()3MEby zQQjzqjWkwfI~;Fuj(Lj=Ug0y`>~C7`w&wzjK(rPw+Hpd~EvQ-ufQOiB4OMpyUKJhw zqEt~jle9d7S~LI~$6Z->J~QJ{Vdn3!c}g9}*KG^Kzr^(7VI5Gk(mHLL{itj_hG?&K4Ws0+T4gLfi3eu$N=`s36geNC?c zm!~}vG6lx9Uf^5M;bWntF<-{p^bruy~f?sk9 zcETAPQZLoJ8JzMMg<-=ju4keY@SY%Wo?u9Gx=j&dfa6LIAB|IrbORLV1-H==Z1zCM zeZcOYpm5>U2fU7V*h;%n`8 zN95QhfD994={1*<2vKLCNF)feKOGk`R#K~G=;rfq}|)s20&MCa65 zUM?xF5!&e0lF%|U!#rD@I{~OsS_?=;s_MQ_b_s=PuWdC)q|UQ&ea)DMRh5>fpQjXe z%9#*x=7{iRCtBKT#H>#v%>77|{4_slZ)XCY{s3j_r{tdpvb#|r|sbS^dU1x70$eJMU!h{Y7Kd{dl}9&vxQl6Jt1a` zHQZrWyY0?!vqf@u-fxU_@+}u(%Wm>0I#KP48tiAPYY!TdW(o|KtVI|EUB9V`CBBNaBLVih7+yMVF|GSoIQD0Jfb{ z!OXq;(>Z?O`1gap(L~bUcp>Lc@Jl-})^=6P%<~~9ywY=$iu8pJ0m*hOPzr~q`23eX zgbs;VOxxENe0UMVeN*>uCn9Gk!4siN-e>x)pIKAbQz!G)TcqIJ0`JBBaX>1-4_XO_-HCS^vr2vjv#7KltDZdyQ{tlWh4$Gm zB>|O1cBDC)yG(sbnc*@w6e%e}r*|IhpXckx&;sQCwGdKH+3oSG-2)Bf#x`@<4ETAr z0My%7RFh6ZLiZ_;X6Mu1YmXx7C$lSZ^}1h;j`EZd6@%JNUe=btBE z%s=Xmo1Ps?8G`}9+6>iaB8bgjUdXT?=trMu|4yLX^m0Dg{m7rpKNJey|EwHI+nN1e zL^>qN%5Fg)dGs4DO~uwIdXImN)QJ*Jhpj7$fq_^`{3fwpztL@WBB}OwQ#Epo-mqMO zsM$UgpFiG&d#)lzEQ{3Q;)&zTw;SzGOah-Dpm{!q7<8*)Ti_;xvV2TYXa}=faXZy? z3y?~GY@kl)>G&EvEijk9y1S`*=zBJSB1iet>0;x1Ai)*`^{pj0JMs)KAM=@UyOGtO z3y0BouW$N&TnwU6!%zS%nIrnANvZF&vB1~P5_d`x-giHuG zPJ;>XkVoghm#kZXRf>qxxEix;2;D1CC~NrbO6NBX!`&_$iXwP~P*c($EVV|669kDO zKoTLZNF4Cskh!Jz5ga9uZ`3o%7Pv`d^;a=cXI|>y;zC3rYPFLQkF*nv(r>SQvD*## z(Vo%^9g`%XwS0t#94zPq;mYGLKu4LU3;txF26?V~A0xZbU4Lmy`)>SoQX^m7fd^*E z+%{R4eN!rIk~K)M&UEzxp9dbY;_I^c} zOc{wlIrN_P(PPqi51k_$>Lt|X6A^|CGYgKAmoI#Li?;Wq%q~q*L7ehZkUrMxW67Jl zhsb~+U?33QS>eqyN{(odAkbopo=Q$Az?L+NZW>j;#~@wCDX?=L5SI|OxI~7!Pli;e zELMFcZtJY3!|=Gr2L4>z8yQ-{To>(f80*#;6`4IAiqUw`=Pg$%C?#1 z_g@hIGerILSU>=P>z{gM|DS91A4cT@PEIB^hSop!uhMo#2G;+tQSpDO_6nOnPWSLU zS;a9m^DFMXR4?*X=}d7l;nXuHk&0|m`NQn%d?8|Ab3A9l9Jh5s120ibWBdB z$5YwsK3;wvp!Kn@)Qae{ef`0#NwlRpQ}k^r>yos_Ne1;xyKLO?4)t_G4eK~wkUS2A&@_;)K0-03XGBzU+5f+uMDxC z(s8!8!RvdC#@`~fx$r)TKdLD6fWEVdEYtV#{ncT-ZMX~eI#UeQ-+H(Z43vVn%Yj9X zLdu9>o%wnWdvzA-#d6Z~vzj-}V3FQ5;axDIZ;i(95IIU=GQ4WuU{tl-{gk!5{l4_d zvvb&uE{%!iFwpymz{wh?bKr1*qzeZb5f6e6m_ozRF&zux2mlK=v_(_s^R6b5lu?_W4W3#<$zeG~Pd)^!4tzhs}-Sx$FJP>)ZGF(hVTH|C3(U zs0PO&*h_ zNA-&qZpTP$$LtIgfiCn07}XDbK#HIXdmv8zdz4TY;ifNIH-0jy(gMSByG2EF~Th#eb_TueZC` zE?3I>UTMpKQ})=C;6p!?G)M6w^u*A57bD?2X`m3X^6;&4%i_m(uGJ3Z5h`nwxM<)H z$I5m?wN>O~8`BGnZ=y^p6;0+%_0K}Dcg|K;+fEi|qoBqvHj(M&aHGqNF48~XqhtU? z^ogwBzRlOfpAJ+Rw7IED8lRbTdBdyEK$gPUpUG}j-M42xDj_&qEAQEtbs>D#dRd7Y z<&TpSZ(quQDHiCFn&0xsrz~4`4tz!CdL8m~HxZM_agu@IrBpyeL1Ft}V$HX_ZqDPm z-f89)pjuEzGdq-PRu`b1m+qBGY{zr_>{6Ss>F|xHZlJj9dt5HD$u`1*WZe)qEIuDSR)%z+|n zatVlhQ?$w#XRS7xUrFE;Y8vMGhQS5*T{ZnY=q1P?w5g$OKJ#M&e??tAmPWHMj3xhS ziGxapy?kn@$~2%ZY;M8Bc@%$pkl%Rvj!?o%agBvpQ-Q61n9kznC4ttrRNQ4%GFR5u zyv%Yo9~yxQJWJSfj z?#HY$y=O~F|2pZs22pu|_&Ajd+D(Mt!nPUG{|1nlvP`=R#kKH zO*s$r_%ss5h1YO7k0bHJ2CXN)Yd6CHn~W!R=SqkWe=&nAZu(Q1G!xgcUilM@YVei@2@a`8he z9@pM`)VB*=e7-MWgLlXlc)t;fF&-AwM{E-EX}pViFn0I0CNw2bNEnN2dj!^4(^zS3 zobUm1uQnpqk_4q{pl*n06=TfK_C>UgurKFjRXsK_LEn};=79`TB12tv6KzwSu*-C8 z;=~ohDLZylHQ|Mpx-?yql>|e=vI1Z!epyUpAcDCp4T|*RV&X`Q$0ogNwy6mFALo^@ z9=&(9txO8V@E!@6^(W0{*~CT>+-MA~vnJULBxCTUW>X5>r7*eXYUT0B6+w@lzw%n> z_VjJ<2qf|(d6jYq2(x$(ZDf!yVkfnbvNmb5c|hhZ^2TV_LBz`9w!e_V*W_(MiA7|= z&EeIIkw*+$Xd!)j8<@_<}A5;~A_>3JT*kX^@}cDoLd>Qj<`Se^wdUa(j0dp+Tl8EptwBm{9OGsdFEq zM`!pjf(Lm(`$e3FLOjqA5LnN5o!}z{ zNf}rJuZh@yUtq&ErjHeGzX4(!luV!jB&;FAP|!R_QHYw#^Z1LwTePAKJ6X&IDNO#; z)#I@Xnnzyij~C@UH~X51JCgQeF0&hTXnuoElz#m{heZRexWc0k4<>0+ClX7%0 zEBqCCld1tD9Zwkr4{?Nor19#E5-YKfB8d?qgR82-Ow2^AuNevly2*tHA|sK!ybYkX zm-sLQH72P&{vEAW6+z~O5d0qd=xW~rua~5a?ymYFSD@8&gV)E5@RNNBAj^C99+Z5Z zR@Pq55mbCQbz+Mn$d_CMW<-+?TU960agEk1J<>d>0K=pF19yN))a~4>m^G&tc*xR+yMD*S=yip-q=H zIlredHpsJV8H(32@Zxc@bX6a21dUV95Th--8pE6C&3F>pk=yv$yd6@Haw;$v4+Fcb zRwn{Qo@0`7aPa2LQOP}j9v>sjOo5Kqvn|`FLizX zB+@-u4Lw|jsvz{p^>n8Vo8H2peIqJJnMN}A)q6%$Tmig7eu^}K2 zrh$X?T|ZMsoh{6pdw1G$_T<`Ds-G=jc;qcGdK4{?dN2-XxjDNbb(7pk|3JUVCU4y; z)?LXR>f+AAu)JEiti_Zy#z5{RgsC}R(@jl%9YZ>zu~hKQ*AxbvhC378-I@{~#%Y`Z zy=a=9YpewPIC+gkEUUwtUL7|RU7=!^Aa}Mk^6uxOgRGA#JXjWLsjFUnix|Mau{hDT z7mn*z1m5g`vP(#tjT0Zy4eAY(br&!RiiXE=ZI!{sE1#^#%x^Z7t1U)b<;%Y}Q9=5v z;wpDCEZ@OE36TWT=|gxigT@VaW9BvHS05;_P(#s z8zI4XFQys}q)<`tkX$WnSarn{3e!s}4(J!=Yf>+Y>cP3f;vr63f2{|S^`_pWc)^5_!R z*(x-fuBxL51@xe!lnDBKi}Br$c$BMZ3%f2Sa6kLabiBS{pq*yj;q|k(86x`PiC{p6 z_bxCW{>Q2BA8~Ggz&0jkrcU+-$ANBsOop*ms>34K9lNYil@}jC;?cYP(m^P}nR6FV zk(M%48Z&%2Rx$A&FhOEirEhY0(dn;-k(qkTU)sFQ`+-ih+s@A8g?r8Pw+}2;35WYf zi}VO`jS`p(tc)$X$a>-#WXoW!phhatC*$}|rk>|wUU71eUJG^$c6_jwX?iSHM@6__ zvV|6%U*$sSXJu9SX?2%M^kK|}a2QJ8AhF{fuXrHZxXsI~O zGKX45!K7p*MCPEQ=gp?eu&#AW*pR{lhQR##P_*{c_DjMGL|3T3-bSJ(o$|M{ytU}> zAV>wq*uE*qFo9KvnA^@juy{x<-u*#2NvkV={Ly}ysKYB-k`K3@K#^S1Bb$8Y#0L0# z`6IkSG&|Z$ODy|VLS+y5pFJx&8tvPmMd8c9FhCyiU8~k6FwkakUd^(_ml8`rnl>JS zZV){9G*)xBqPz^LDqRwyS6w86#D^~xP4($150M)SOZRe9sn=>V#aG0Iy(_^YcPpIz8QYM-#s+n% z@Jd?xQq?Xk6=<3xSY7XYP$$yd&Spu{A#uafiIfy8gRC`o0nk{ezEDjb=q_qRAlR1d zFq^*9Gn)yTG4b}R{!+3hWQ+u3GT~8nwl2S1lpw`s0X_qpxv)g+JIkVKl${sYf_nV~B>Em>M;RlqGb5WVil(89 zs=ld@|#;dq1*vQGz=7--Br-|l) zZ%Xh@v8>B7P?~}?Cg$q9_={59l%m~O&*a6TKsCMAzG&vD>k2WDzJ6!tc!V)+oxF;h zJH;apM=wO?r_+*#;ulohuP=E>^zon}a$NnlcQ{1$SO*i=jnGVcQa^>QOILc)e6;eNTI>os=eaJ{*^DE+~jc zS}TYeOykDmJ=6O%>m`i*>&pO_S;qMySJIyP=}4E&J%#1zju$RpVAkZbEl+p%?ZP^C z*$$2b4t%a(e+%>a>d_f_<JjxI#J1x;=hPd1zFPx=6T$;;X1TD*2(edZ3f46zaAoW>L53vS_J*N8TMB|n+;LD| zC=GkQPpyDY#Am4l49chDv*gojhRj_?63&&8#doW`INATAo(qY#{q}%nf@eTIXmtU< zdB<7YWfyCmBs|c)cK>1)v&M#!yNj#4d$~pVfDWQc_ke1?fw{T1Nce_b`v|Vp5ig(H zJvRD^+ps46^hLX;=e2!2e;w9y1D@!D$c@Jc&%%%IL=+xzw55&2?darw=9g~>P z9>?Kdc$r?6c$m%x2S$sdpPl>GQZ{rC9mPS63*qjCVa?OIBj!fW zm|g?>CVfGXNjOfcyqImXR_(tXS(F{FcoNzKvG5R$IgGaxC@)i(e+$ME}vPVIhd|mx2IIE+f zM?9opQHIVgBWu)^A|RzXw!^??S!x)SZOwZaJkGjc<_}2l^eSBm!eAJG9T>EC6I_sy z?bxzDIAn&K5*mX)$RQzDA?s)-no-XF(g*yl4%+GBf`##bDXJ==AQk*xmnatI;SsLp zP9XTHq5mmS=iWu~9ES>b%Q=1aMa|ya^vj$@qz9S!ih{T8_PD%Sf_QrNKwgrXw9ldm zHRVR98*{C?_XNpJn{abA!oix_mowRMu^2lV-LPi;0+?-F(>^5#OHX-fPED zCu^l7u3E%STI}c4{J2!)9SUlGP_@!d?5W^QJXOI-Ea`hFMKjR7TluLvzC-ozCPn1`Tpy z!vlv@_Z58ILX6>nDjTp-1LlFMx~-%GA`aJvG$?8*Ihn;mH37eK**rmOEwqegf-Ccx zrIX4;{c~RK>XuTXxYo5kMiWMy)!IC{*DHG@E$hx?RwP@+wuad(P1{@%tRkyJRqD)3 zMHHHZ4boqDn>-=DgR5VlhQTpfVy182Gk;A_S8A1-;U1RR>+$62>(MUx@Nox$vTjHq z%QR=j!6Gdyb5wu7y(YUktwMuW5<@jl?m4cv4BODiT5o8qVdC0MBqGr@-YBIwnpZAY znX9(_uQjP}JJ=!~Ve9#5I~rUnN|P_3D$LqZcvBnywYhjlMSFHm`;u9GPla{5QD7(7*6Tb3Svr8;(nuAd81q$*uq6HC_&~je*Ca7hP4sJp0av{M8480wF zxASi7Qv+~@2U%Nu1Ud;s-G4CTVWIPyx!sg&8ZG0Wq zG_}i3C(6_1>q3w!EH7$Kwq8uBp2F2N7}l65mk1p*9v0&+;th=_E-W)E;w}P(j⁢ zv5o9#E7!G0XmdzfsS{efPNi`1b44~SZ4Z8fuX!I}#8g+(wxzQwUT#Xb2(tbY1+EUhGKoT@KEU9Ktl>_0 z%bjDJg;#*gtJZv!-Zs`?^}v5eKmnbjqlvnSzE@_SP|LG_PJ6CYU+6zY6>92%E+ z=j@TZf-iW4(%U{lnYxQA;7Q!b;^brF8n0D>)`q5>|WDDXLrqYU_tKN2>=#@~OE7grMnNh?UOz-O~6 z6%rHy{#h9K0AT+lDC7q4{hw^|q6*Ry;;L%Q@)Ga}$60_q%D)rv(CtS$CQbpq9|y1e zRSrN4;$Jyl{m5bZw`$8TGvb}(LpY{-cQ)fcyJv7l3S52TLXVDsphtv&aPuDk1OzCA z4A^QtC(!11`IsNx_HnSy?>EKpHJWT^wmS~hc^p^zIIh@9f6U@I2 zC=Mve{j2^)mS#U$e{@Q?SO6%LDsXz@SY+=cK_QMmXBIU)j!$ajc-zLx3V60EXJ!qC zi<%2x8Q24YN+&8U@CIlN zrZkcT9yh%LrlGS9`G)KdP(@9Eo-AQz@8GEFWcb7U=a0H^ZVbLmz{+&M7W(nXJ4sN8 zJLR7eeK(K8`2-}j(T7JsO`L!+CvbueT%izanm-^A1Dn{`1Nw`9P?cq;7no+XfC`K(GO9?O^5zNIt4M+M8LM0=7Gz8UA@Z0N+lg+cX)NfazRu z5D)~HA^(u%w^cz+@2@_#S|u>GpB+j4KzQ^&Wcl9f z&hG#bCA(Yk0D&t&aJE^xME^&E-&xGHhXn%}psEIj641H+Nl-}boj;)Zt*t(4wZ5DN z@GXF$bL=&pBq-#vkTkh>7hl%K5|3 z{`Vn9b$iR-SoGENp}bn4;fR3>9sA%X2@1L3aE9yTra;Wb#_`xWwLSLdfu+PAu+o3| zGVnpzPr=ch{uuoHjtw7+_!L_2;knQ!DuDl0R`|%jr+}jFzXtrHIKc323?JO{l&;VF z*L1+}JU7%QJOg|5|Tc|D8fN zJORAg=_vsy{ak|o);@)Yh8Lkcg@$FG3k@ep36BRa^>~UmnRPziS>Z=`Jb2x*Q#`%A zU*i3&Vg?TluO@X0O;r2Jl6LKLUOVhSqg1*qOt^|8*c7 zo(298@+r$k_wQNGHv{|$tW(T8L+4_`FQ{kEW5Jgg{yf7ey4ss_(SNKfz(N9lx&a;< je(UuV8hP?p&}TPdm1I$XmG#(RzlD&B2izSj9sl%y5~4qc literal 0 HcmV?d00001 diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..ec97aa7f8d --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https://services.gradle.org/distributions/gradle-8.9-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew new file mode 100755 index 0000000000..b740cf1339 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat new file mode 100644 index 0000000000..7101f8e467 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py index af13b67799..310faf32da 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py @@ -44,7 +44,7 @@ import urllib.parse import shutil import util -from abc import ABC, abstractmethod +import sys is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true" is_gradle_enabled = 'ENABLE_GRADLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_GRADLE_STANDALONE_UNITTESTS'] == "true" @@ -92,6 +92,19 @@ def replace_in_file(file, str, replace_str): with open(file, "w") as f: f.write(contents.replace(str, replace_str)) +def patch_properties_file(properties_file, distribution_url_override): + if distribution_url_override: + new_lines = [] + with(open(properties_file)) as f: + while line := f.readline(): + line.strip() + if not line.startswith("#") and "distributionUrl" in line: + new_lines.append(f"distributionUrl={distribution_url_override}\n") + else: + new_lines.append(line) + with(open(properties_file, "w")) as f: + f.writelines(new_lines) + class PolyglotAppTestBase(unittest.TestCase): def setUpClass(self): if not is_enabled and not is_gradle_enabled: @@ -169,66 +182,66 @@ def append(file, txt): with open(file, "a") as f: f.write(txt) -class PolyglotAppGradleTestBase(PolyglotAppTestBase, ABC): +class PolyglotAppGradleTestBase(PolyglotAppTestBase): + def setUpClass(self): + super().setUpClass() + self.test_prj_path = os.path.join(os.path.dirname(__file__), "gradle", "gradle-test-project") - @abstractmethod def target_dir_name_sufix(self, target_dir): pass - @abstractmethod def copy_build_files(self, target_dir): pass - @abstractmethod def packages_termcolor(self, build_file): pass - @abstractmethod def packages_termcolor_ujson(self): pass - @abstractmethod def packages_termcolor_resource_dir(self, resources_dir): pass - @abstractmethod def empty_home_includes(self): pass - @abstractmethod def home_includes(self): pass - @abstractmethod def empty_packages(self): pass - def generate_app(self, tmpdir, target_dir): - src_prj_path = os.path.join(os.path.dirname(__file__), "gradle", "gradle-test-project") - for root, dirs, files in os.walk(src_prj_path): + def generate_app(self, target_dir): + shutil.copytree(self.test_prj_path, target_dir) + for root, dirs, files in os.walk(target_dir): for file in files: - source_file = os.path.join(root, file) if file.endswith(".j"): - file = file[0:len(file)- 1] + "java" - target_root = os.path.join(target_dir, root[len(src_prj_path) + 1:]) - target_file = os.path.join(target_root, file) - os.makedirs(os.path.dirname(target_file), exist_ok=True) - shutil.copyfile(source_file, target_file) + shutil.move(os.path.join(root, file), os.path.join(root, file[0:len(file)- 1] + "java")) + + patch_properties_file(os.path.join(target_dir, "gradle", "wrapper", "gradle-wrapper.properties"), self.env.get("GRADLE_DISTRIBUTION_URL_OVERRIDE")) self.copy_build_files(target_dir) @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") - def test_generated_gradle_app(self): - with tempfile.TemporaryDirectory() as tmpdir: + def test_gradle_generated_app(self): + with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "generated_app_gradle" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file = os.path.join(target_dir, self.build_file_name) append(build_file, self.packages_termcolor()) gradlew_cmd = util.get_gradle_wrapper(target_dir, self.env) # build + cmd = gradlew_cmd + ["build"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + cmd = gradlew_cmd + ["nativeCompile"] + # gradle needs jdk <= 22, but it looks like the 'gradle nativeCompile' cmd does not complain if higher, + # which is fine, because we need to build the native binary with a graalvm build + # and the one we have set in JAVA_HOME is at least jdk24 + # => run without gradle = True out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) util.check_ouput("BUILD SUCCESS", out) @@ -254,7 +267,7 @@ def test_generated_gradle_app(self): # rebuild and exec cmd = gradlew_cmd + ["build", "run"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("hello java", out) @@ -262,10 +275,10 @@ def test_generated_gradle_app(self): util.check_ouput("java.lang.NoClassDefFoundError", out, False) @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") - def test_generated_gradle_app_external_resources(self): + def test_gradle_generated_app_external_resources(self): with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "generated_gradle_app_external_resources" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file = os.path.join(target_dir, self.build_file_name) # patch project to use external directory for resources @@ -282,7 +295,7 @@ def test_generated_gradle_app_external_resources(self): "package org.example;\nimport java.nio.file.Path;") replace_in_file(os.path.join(target_dir, "src", "main", "java", "org", "example", "GraalPy.java"), "GraalPyResources.createContext()", - "GraalPyResources.contextBuilder(Path.of(\"" + resources_dir + "\")).build()") + "GraalPyResources.contextBuilder(Path.of(\"python-resources\")).build()") # patch build.gradle append(build_file, self.packages_termcolor_resource_dir(resources_dir)) @@ -290,7 +303,15 @@ def test_generated_gradle_app_external_resources(self): # build gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) - cmd = gradle_cmd + ["clean", "nativeCompile"] + cmd = gradle_cmd + ["clean", "build"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + + # gradle needs jdk <= 22, but it looks like the 'gradle nativeCompile' cmd does not complain if higher, + # which is fine, because we need to build the native binary with a graalvm build + # and the one we have set in JAVA_HOME is at least jdk24 + # => run without gradle = True + cmd = gradle_cmd + ["nativeCompile"] out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) util.check_ouput("BUILD SUCCESS", out) @@ -301,7 +322,7 @@ def test_generated_gradle_app_external_resources(self): # 2.) check java build and exec cmd = gradle_cmd + ["run"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("hello java", out) @@ -309,7 +330,7 @@ def test_generated_gradle_app_external_resources(self): def test_gradle_fail_without_graalpy_dep(self): with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "gradle_fail_without_graalpy_dep" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file = os.path.join(target_dir, self.build_file_name) append(build_file, GRAALPY_EMPTY) @@ -320,14 +341,14 @@ def test_gradle_fail_without_graalpy_dep(self): "// implementation(\"org.graalvm.python:python-community:24.2.0\")") cmd = gradle_cmd + ["graalPyResources"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("Missing GraalPy dependency. Please add to your build.gradle either org.graalvm.polyglot:python-community or org.graalvm.polyglot:python", out) @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") def test_gradle_gen_launcher_and_venv(self): with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "gradle_gen_launcher_and_venv" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file = os.path.join(target_dir, self.build_file_name) @@ -336,7 +357,7 @@ def test_gradle_gen_launcher_and_venv(self): append(build_file, self.packages_termcolor_ujson()) cmd = gradle_cmd + ["graalPyResources"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("-m venv", out) util.check_ouput("-m ensurepip",out) util.check_ouput("ujson", out) @@ -344,18 +365,18 @@ def test_gradle_gen_launcher_and_venv(self): # run again and assert that we do not regenerate the venv cmd = gradle_cmd + ["graalPyResources"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("-m venv", out, False) util.check_ouput("-m ensurepip", out, False) util.check_ouput("ujson", out, False) util.check_ouput("termcolor", out, False) # remove ujson pkg from plugin config and check if unistalled - shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) + self.copy_build_files(target_dir) append(build_file, self.packages_termcolor()) cmd = gradle_cmd + ["graalPyResources"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("-m venv", out, False) util.check_ouput("-m ensurepip", out, False) util.check_ouput("Uninstalling ujson", out) @@ -367,10 +388,10 @@ def check_tagfile(self, home, expected): assert lines == expected, "expected tagfile " + str(expected) + ", but got " + str(lines) @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") - def test_check_home(self): + def test_gradle_check_home(self): with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "check_home_test" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file_template = os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name) build_file = os.path.join(target_dir, self.build_file_name) @@ -379,7 +400,7 @@ def test_check_home(self): process_resources_cmd = gradle_cmd + ["graalPyResources"] # 1. process-resources with no pythonHome config - out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("Copying std lib to ", out) @@ -394,25 +415,25 @@ def test_check_home(self): self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) # 2. process-resources with empty pythonHome - shutil.copyfile(build_file_template, build_file) + self.copy_build_files(target_dir) append(build_file, GRAALPY_EMPTY_HOME) - out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("Copying std lib to ", out, False) self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) # 3. process-resources with empty pythonHome includes and excludes - shutil.copyfile(build_file_template, build_file) + self.copy_build_files(target_dir) append(build_file, self.empty_home_includes()) - out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("Copying std lib to ", out, False) self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) # 4. process-resources with pythonHome includes and excludes - shutil.copyfile(build_file_template, build_file) + self.copy_build_files(target_dir) append(build_file, self.home_includes()) - out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) util.check_ouput("Deleting GraalPy home due to changed includes or excludes", out) util.check_ouput("Copying std lib to ", out) @@ -420,7 +441,7 @@ def test_check_home(self): # 5. check fileslist.txt # XXX build vs graalPyVFSFilesList task? - out, return_code = util.run_cmd(gradle_cmd + ["build"], self.env, cwd=target_dir) + out, return_code = util.run_cmd(gradle_cmd + ["build"], self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) fl_path = os.path.join(target_dir, "build", "resources", "main", VFS_PREFIX, "fileslist.txt") with open(fl_path) as f: @@ -436,19 +457,28 @@ def test_check_home(self): assert not line.endswith("html/__init__.py"), f"expected line to end with html/__init__.py, but was '{line}''" @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") - def test_empty_packages(self): + def test_gradle_empty_packages(self): with tempfile.TemporaryDirectory() as tmpdir: target_dir = os.path.join(str(tmpdir), "empty_packages_test" + self.target_dir_name_sufix()) - self.generate_app(tmpdir, target_dir) + self.generate_app(target_dir) build_file = os.path.join(target_dir, self.build_file_name) append(build_file, self.empty_packages()) gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) cmd = gradle_cmd + ["graalPyResources"] - out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) util.check_ouput("BUILD SUCCESS", out) +def print_file(file): + print("\n====", file, " ==========================================================================") + with open(file) as f: + while line := f.readline(): + if line.endswith("\n"): + line = line[:len(line) - 1] + print(line) + print("\n========================================================================================") + class PolyglotAppGradleGroovyTest(PolyglotAppGradleTestBase): def setUpClass(self): @@ -460,8 +490,21 @@ def target_dir_name_sufix(self): return "_groovy" def copy_build_files(self, target_dir): - shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) - shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), os.path.join(target_dir, self.settings_file_name)) + build_file = os.path.join(target_dir, self.build_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), build_file) + settings_file = os.path.join(target_dir, self.settings_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), settings_file) + if custom_repos := os.environ.get("MAVEN_REPO_OVERRIDE"): + mvn_repos = "" + for idx, custom_repo in enumerate(custom_repos.split(",")): + mvn_repos += f"maven {{ url \"{custom_repo}\" }}\n " + replace_in_file(build_file, + "repositories {", f"repositories {{\n mavenLocal()\n {mvn_repos}") + replace_in_file(settings_file, + "repositories {", f"repositories {{\n {mvn_repos}") + + #print_file(build_file) + #print_file(settings_file) def packages_termcolor(self): return """ @@ -478,6 +521,7 @@ def packages_termcolor_ujson(self): """ def packages_termcolor_resource_dir(self, resources_dir): + resources_dir = resources_dir if 'win32' != sys.platform else resources_dir.replace("\\", "\\\\") return f""" graalPy {{ packages = ["termcolor"] @@ -523,8 +567,20 @@ def target_dir_name_sufix(self): return "_kotlin" def copy_build_files(self, target_dir): - shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), os.path.join(target_dir, self.build_file_name)) - shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), os.path.join(target_dir, self.settings_file_name)) + build_file = os.path.join(target_dir, self.build_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), build_file) + settings_file = os.path.join(target_dir, self.settings_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), settings_file) + if custom_repos := os.environ.get("MAVEN_REPO_OVERRIDE"): + mvn_repos = "" + for idx, custom_repo in enumerate(custom_repos.split(",")): + mvn_repos += f"maven(url=\"{custom_repo}\")\n " + + replace_in_file(build_file, "repositories {", f"repositories {{\n mavenLocal()\n {mvn_repos}") + replace_in_file(settings_file, "repositories {", f"repositories {{\n {mvn_repos}") + + #print_file(build_file) + #print_file(settings_file) def packages_termcolor(self): return """ @@ -542,6 +598,7 @@ def packages_termcolor_ujson(self): """ def packages_termcolor_resource_dir(self, resources_dir): + resources_dir = resources_dir if 'win32' != sys.platform else resources_dir.replace("\\", "\\\\") return f""" graalPy {{ packages.add("termcolor") @@ -598,25 +655,11 @@ def generate_app(self, tmpdir, target_dir, target_name, pom_template=None): util.patch_pom_repositories(os.path.join(target_dir, "pom.xml")) - distribution_url_override = self.env.get("MAVEN_DISTRIBUTION_URL_OVERRIDE") - if distribution_url_override: - mvnw_dir = os.path.join(os.path.dirname(__file__), "mvnw") - - shutil.copy(os.path.join(mvnw_dir, "mvnw"), os.path.join(target_dir, "mvnw")) - shutil.copy(os.path.join(mvnw_dir, "mvnw.cmd"), os.path.join(target_dir, "mvnw.cmd")) - shutil.copytree(os.path.join(mvnw_dir, ".mvn"), os.path.join(target_dir, ".mvn")) - - mvnw_properties = os.path.join(target_dir, ".mvn", "wrapper", "maven-wrapper.properties") - new_lines = [] - with(open(mvnw_properties)) as f: - while line := f.readline(): - line.strip() - if not line.startswith("#") and "distributionUrl" in line: - new_lines.append(f"distributionUrl={distribution_url_override}\n") - else: - new_lines.append(line) - with(open(mvnw_properties, "w")) as f: - f.writelines(new_lines) + mvnw_dir = os.path.join(os.path.dirname(__file__), "mvnw") + shutil.copy(os.path.join(mvnw_dir, "mvnw"), os.path.join(target_dir, "mvnw")) + shutil.copy(os.path.join(mvnw_dir, "mvnw.cmd"), os.path.join(target_dir, "mvnw.cmd")) + shutil.copytree(os.path.join(mvnw_dir, ".mvn"), os.path.join(target_dir, ".mvn")) + patch_properties_file(os.path.join(target_dir, ".mvn", "wrapper", "maven-wrapper.properties"), self.env.get("MAVEN_DISTRIBUTION_URL_OVERRIDE")) @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true") def test_generated_app(self): diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py index 9254c71b79..ab6ff02970 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py @@ -43,21 +43,32 @@ import sys MAVEN_VERSION = "3.9.8" -GRADLE_VERSION = "8.8" +GRADLE_VERSION = "8.9" -def run_cmd(cmd, env, cwd=None, print_out=False): +def run_cmd(cmd, env, cwd=None, print_out=False, gradle=False): out = [] out.append(f"Executing:\n {cmd=}\n") - process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True, errors='backslashreplace') - if print_out: - print("============== output =============") - for line in iter(process.stdout.readline, ""): - out.append(line) + prev_java_home = None + if gradle: + gradle_java_home = env.get("GRADLE_JAVA_HOME") + assert gradle_java_home, "in order to run standalone gradle tests, the 'GRADLE_JAVA_HOME' env var has to be set to a jdk <= 22" + prev_java_home = env["JAVA_HOME"] + env["JAVA_HOME"] = env["GRADLE_JAVA_HOME"] + + try: + process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True, errors='backslashreplace') if print_out: - print(line, end="") - if print_out: - print("\n========== end of output ==========") - return "".join(out), process.wait() + print("============== output =============") + for line in iter(process.stdout.readline, ""): + out.append(line) + if print_out: + print(line, end="") + if print_out: + print("\n========== end of output ==========") + return "".join(out), process.wait() + finally: + if prev_java_home: + env["JAVA_HOME"] = prev_java_home def check_ouput(txt, out, contains=True): if contains and txt not in out: @@ -84,10 +95,6 @@ def get_mvn_wrapper(project_dir, env): return mvn_cmd def get_gradle_wrapper(project_dir, env): - gradle = shutil.which('gradle') - cmd = [gradle, "wrapper", f"--gradle-version={GRADLE_VERSION}"] - out, return_code = run_cmd(cmd, env, cwd=project_dir) - check_ouput("BUILD SUCCESS", out) gradle_cmd = [os.path.abspath(os.path.join(project_dir, "gradlew" if 'win32' != sys.platform else "gradlew.bat"))] cmd = gradle_cmd + ["--version"] out, return_code = run_cmd(cmd, env, cwd=project_dir) diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 847dc1556c..9c63f1ac43 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1536,9 +1536,11 @@ def graalpython_gate_runner(args, tasks): standalone_home = graalpy_standalone_home('jvm') mvn_repo_path, version, env = deploy_local_maven_repo() + # in order to run gradle we need a jdk <= 22 + env['GRADLE_JAVA_HOME'] = env.get('JAVA_HOME') + env['ENABLE_STANDALONE_UNITTESTS'] = 'true' - # TODO disabled until gradle available on gate - # env['ENABLE_ENABLE_GRADLE_STANDALONE_UNITTESTS'] = 'true' + env['ENABLE_GRADLE_STANDALONE_UNITTESTS'] = 'true' env['ENABLE_JBANG_INTEGRATION_UNITTESTS'] ='true' env['JAVA_HOME'] = gvm_jdk env['PYTHON_STANDALONE_HOME'] = standalone_home @@ -1553,6 +1555,10 @@ def graalpython_gate_runner(args, tasks): if "distributionUrl" in urls: env["MAVEN_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"]) + urls = get_wrapper_urls("graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties", ["distributionUrl"]) + if "distributionUrl" in urls: + env["GRADLE_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"]) + env["org.graalvm.maven.downloader.version"] = version env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(mvn_repo_path).as_uri()}/" From 8139937f86c7859359fcf82cf700368fb33175a0 Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Thu, 22 Aug 2024 15:43:09 +0200 Subject: [PATCH 15/16] changelog entry for gradle plugin --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1559248bfd..9d91cda6c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ language runtime. The main focus is on user-observable behavior of the engine. ## Version 24.2.0 * Updated developer metadata of Maven artifacts. +* Added gradle plugin for polyglot embedding of Python packages into Java. ## Version 24.1.0 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'. From 9d40408e79a9ad9e277d2eaa8774ff861895140f Mon Sep 17 00:00:00 2001 From: Tomas Stupka Date: Wed, 28 Aug 2024 16:19:45 +0200 Subject: [PATCH 16/16] rename graalpy-gradle-plugin to org.graalvm.python.gradle.plugin, so that it is recognizable to by gradle projects as org.graalvm.python without additional configuration --- .../tests/standalone/gradle/build/build.gradle | 2 +- .../standalone/gradle/build/build.gradle.kts | 2 +- .../standalone/gradle/build/settings.gradle | 15 +++++---------- .../standalone/gradle/build/settings.gradle.kts | 13 +++++-------- .../pom.xml | 16 ++++++++-------- .../org/graalvm/python/GraalPyGradlePlugin.java | 0 .../java/org/graalvm/python/GradleLogger.java | 0 .../org/graalvm/python/dsl/GraalPyExtension.java | 0 .../org/graalvm/python/dsl/PythonHomeInfo.java | 0 .../org/graalvm/python/tasks/MetaInfTask.java | 2 +- .../org/graalvm/python/tasks/ResourcesTask.java | 0 .../graalvm/python/tasks/VFSFilesListTask.java | 0 .../gradle-plugins/org.graalvm.python.properties | 0 mx.graalpython/suite.py | 2 +- 14 files changed, 22 insertions(+), 30 deletions(-) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/pom.xml (95%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/GradleLogger.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/tasks/MetaInfTask.java (98%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/tasks/ResourcesTask.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java (100%) rename graalpython/{graalpy-gradle-plugin => org.graalvm.python.gradle.plugin}/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties (100%) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle index b03dcb7441..ac804e7fcb 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle @@ -1,6 +1,6 @@ plugins { id "application" - id "org.graalvm.python" + id 'org.graalvm.python' version '24.2.0' id "org.graalvm.buildtools.native" version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts index 26230d7185..119e1482b1 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts @@ -1,6 +1,6 @@ plugins { application - id("org.graalvm.python") + id("org.graalvm.python") version "24.2.0" id("org.graalvm.buildtools.native") version "0.10.2" } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle index 94a40b69a8..cd2662dcd4 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle @@ -1,12 +1,7 @@ -rootProject.name = "graalpy-gradle-test-project" - -buildscript { +pluginManagement { repositories { - maven { - url "https://repo.gradle.org/gradle/libs-releases/" - } - } - dependencies { - classpath "org.graalvm.python:graalpy-gradle-plugin:24.2.0" + gradlePluginPortal() } -} \ No newline at end of file +} + +rootProject.name = "graalpy-gradle-test-project" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts index 3353bd017a..cd2662dcd4 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts @@ -1,10 +1,7 @@ -rootProject.name = "graalpy-gradle-test-project" - -buildscript { +pluginManagement { repositories { - maven(url="https://repo.gradle.org/gradle/libs-releases/") - } - dependencies { - classpath("org.graalvm.python:graalpy-gradle-plugin:24.2.0") + gradlePluginPortal() } -} \ No newline at end of file +} + +rootProject.name = "graalpy-gradle-test-project" diff --git a/graalpython/graalpy-gradle-plugin/pom.xml b/graalpython/org.graalvm.python.gradle.plugin/pom.xml similarity index 95% rename from graalpython/graalpy-gradle-plugin/pom.xml rename to graalpython/org.graalvm.python.gradle.plugin/pom.xml index 4e50ced5c2..c7828c84c8 100644 --- a/graalpython/graalpy-gradle-plugin/pom.xml +++ b/graalpython/org.graalvm.python.gradle.plugin/pom.xml @@ -45,7 +45,7 @@ SOFTWARE. 4.0.0 org.graalvm.python - graalpy-gradle-plugin + org.graalvm.python.gradle.plugin jar 24.2.0 @@ -59,6 +59,7 @@ SOFTWARE. UTF-8 24.2.0 6.1.1 + 2.0.13 @@ -75,7 +76,6 @@ SOFTWARE. - org.codehaus.groovy @@ -83,11 +83,6 @@ SOFTWARE. 3.0.22 provided - - org.gradle - gradle-tooling-api - 8.8 - org.gradle gradle-core @@ -142,7 +137,12 @@ SOFTWARE. ${gradle.version} provided - + + org.slf4j + slf4j-simple + ${slf4j.version} + provided + org.graalvm.python python-embedding-tools diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GradleLogger.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/GradleLogger.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GradleLogger.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java similarity index 98% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java index 0815faea5c..2b3c4a927f 100644 --- a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java @@ -50,7 +50,7 @@ public abstract class MetaInfTask extends DefaultTask { - private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "graalpy-gradle-plugin"; + private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "org.graalvm.python.gradle.plugin"; @OutputDirectory public abstract DirectoryProperty getManifestOutputDir(); diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java rename to graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java diff --git a/graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties b/graalpython/org.graalvm.python.gradle.plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties similarity index 100% rename from graalpython/graalpy-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties rename to graalpython/org.graalvm.python.gradle.plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 702b778d59..2f334054de 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -1466,7 +1466,7 @@ "tag": ["default", "public"], }, }, - "graalpy-gradle-plugin": { + "org.graalvm.python.gradle.plugin": { "class": "MavenProject", "subDir": "graalpython", "noMavenJavadoc": True,