From 1fd121455f5caf5c2d3b5bc3125fa6f67d8ccc59 Mon Sep 17 00:00:00 2001 From: Jason Montleon Date: Thu, 7 Sep 2023 15:00:01 -0400 Subject: [PATCH] :ghost: Fix maven index and update it weekly (#63) Signed-off-by: Jason Montleon --- .../workflows/multi-arch_images_build.yaml | 9 ------- .github/workflows/update_maven_index.yaml | 25 +++++++++++++++++ Dockerfile | 5 ---- hack/Dockerfile | 27 +++++++++++++++++++ hack/export.java | 23 ++++++++++++++++ 5 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/update_maven_index.yaml create mode 100644 hack/Dockerfile create mode 100644 hack/export.java diff --git a/.github/workflows/multi-arch_images_build.yaml b/.github/workflows/multi-arch_images_build.yaml index f43ed43..492ed20 100644 --- a/.github/workflows/multi-arch_images_build.yaml +++ b/.github/workflows/multi-arch_images_build.yaml @@ -20,23 +20,14 @@ jobs: strategy: fail-fast: false steps: - - name: Install pigz - shell: bash - run: sudo apt-get install -y pigz - name: Checkout Push to Registry action uses: konveyor/release-tools/build-push-quay@main with: architectures: "amd64, arm64, ppc64le, s390x" containerfile: "./Dockerfile" - extra-args: "--build-arg BUILDENV=github" image_name: "jdtls-server-base" image_namespace: "konveyor" image_registry: "quay.io" quay_publish_robot: ${{ secrets.QUAY_PUBLISH_ROBOT }} quay_publish_token: ${{ secrets.QUAY_PUBLISH_TOKEN }} - pre_build_cmd: | - curl -s -o nexus-maven-repository-index.gz https://repo.maven.apache.org/maven2/.index/nexus-maven-repository-index.gz - unpigz nexus-maven-repository-index.gz - cat nexus-maven-repository-index | parallel -j150% --pipe --block 100M --cat grep -oaP "'[a-z][a-zA-Z0-9_.-]+\|[a-zA-Z][a-zA-Z0-9_.-]+\|([a-zA-Z0-9_.-]+\|)?(sources|pom|jar|maven-plugin|ear|ejb|ejb-client|java-source|rar|war)\|(jar|war|ear|pom|war|rar)'" {} | cut -d'|' -f1 | sed 's/$/.*/' | sort | uniq > hack/maven.default.index - rm -rf nexus-maven-repository-index ref: ${{ github.ref }} diff --git a/.github/workflows/update_maven_index.yaml b/.github/workflows/update_maven_index.yaml new file mode 100644 index 0000000..2f7fe1a --- /dev/null +++ b/.github/workflows/update_maven_index.yaml @@ -0,0 +1,25 @@ +name: 'Generate Maven Index and Commit' +on: + workflow_dispatch: + schedule: + - cron: '0 3 * * 0' + +jobs: + build_and_commit: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install podman + shell: bash + run: sudo apt-get install -y podman + - name: + run: | + podman build -f hack/Dockerfile -t export hack/ + podman cp $(podman create export):/unpack/maven.default.index ./hack + - name: Commit Updated maven.default.index update + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: ":ghost: Update Maven Index" diff --git a/Dockerfile b/Dockerfile index d17741a..c4d2aa6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,7 @@ RUN curl -s -o jdtls.tar.gz https://download.eclipse.org/jdtls/milestones/1.16.0 rm -rf jdtls.tar.gz FROM registry.access.redhat.com/ubi9/ubi AS maven-index -ARG BUILDENV=local COPY hack/maven.default.index /maven.default.index -RUN if [ $BUILDENV == "local" ]; then curl -s -o nexus-maven-repository-index.gz https://repo.maven.apache.org/maven2/.index/nexus-maven-repository-index.gz &&\ - zgrep -oaP '[a-z][a-zA-Z0-9_.-]+\|[a-zA-Z][a-zA-Z0-9_.-]+\|([a-zA-Z0-9_.-]+\|)?(sources|pom|jar|maven-plugin|ear|ejb|ejb-client|java-source|rar|war)\|(jar|war|ear|pom|war|rar)' nexus-maven-repository-index.gz | cut -d'|' -f1 | sed 's/$/.*/' | sort | uniq > /maven.default.index &&\ - rm -rf nexus-maven-repository-index.gz; fi - FROM registry.access.redhat.com/ubi9/ubi AS fernflower RUN dnf install -y maven-openjdk17 wget --setopt=install_weak_deps=False && dnf clean all && rm -rf /var/cache/dnf RUN wget --quiet https://github.com/JetBrains/intellij-community/archive/refs/tags/idea/231.9011.34.tar.gz -O intellij-community.tar && tar xf intellij-community.tar intellij-community-idea-231.9011.34/plugins/java-decompiler/engine && rm -rf intellij-community.tar diff --git a/hack/Dockerfile b/hack/Dockerfile new file mode 100644 index 0000000..f9200ef --- /dev/null +++ b/hack/Dockerfile @@ -0,0 +1,27 @@ +FROM registry.access.redhat.com/ubi9/ubi +RUN dnf module enable -y maven:3.8 \ + && dnf clean all \ + && dnf -y update \ + && dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ + && dnf -y install maven-openjdk17 git parallel pigz \ + && dnf clean all +RUN git clone https://github.com/apache/maven-indexer.git +RUN cd maven-indexer/indexer-cli && mvn install -Denforcer.skip=true -Dmaven.test.skip=true +RUN mkdir -p /unpack/index +COPY export.java /unpack/export.java +RUN cd /unpack/index \ + && curl -s -o nexus-maven-repository-index.gz https://repo.maven.apache.org/maven2/.index/nexus-maven-repository-index.gz \ + && java -jar $(find /root/.m2 -name indexer-cli-*-cli.jar) -u nexus-maven-repository-index.gz \ + && rm nexus-maven-repository-index.gz \ + && cd /unpack \ + && java -classpath $(find /root/.m2 -name lucene-core*.jar) export.java | pigz -k -p$(nproc) > export.gz \ + && rm -rf index \ + && unpigz -p$(nproc) export.gz \ + && cat export | \ + parallel -j150% --pipe --block 100M --cat \ + grep -oaP "'[a-z][a-zA-Z0-9_.-]+\|[a-zA-Z][a-zA-Z0-9_.-]+\|([a-zA-Z0-9_.-]+\|)?(sources|pom|jar|maven-plugin|ear|ejb|ejb-client|java-source|rar|war)\|(jar|war|ear|pom|war|rar)'" {} | \ + cut -d'|' -f1 | \ + sed 's/$/.*/' | \ + sort | \ + uniq > maven.default.index \ + && rm -rf export diff --git a/hack/export.java b/hack/export.java new file mode 100644 index 0000000..b035855 --- /dev/null +++ b/hack/export.java @@ -0,0 +1,23 @@ +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class Export { + public static void main(String[] args) throws Exception { + Path path = Paths.get("index"); + Directory index = FSDirectory.open(path); + IndexReader reader = DirectoryReader.open(index); + + int num = reader.numDocs(); + for ( int i = 0; i < num; i++) + { + Document d = reader.document( i); + System.out.println( "d=" +d); + } + reader.close(); + } +}