feat: deploy Docker registry on Kubernetes

.github/workflows/e2e-tests.yml

@@ -101,7 +101,7 @@ jobs:
           minikube version: v1.25.2
           kubernetes version: ${{ matrix.kubernetes }}
           github token: ${{ secrets.GITHUB_TOKEN }}
-          start args: --force
+          start args: '--force --insecure-registry "0.0.0.0/0"'
       - name: Harden Runner
         uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a
         with:
@@ -139,83 +139,8 @@ jobs:
             us-east4-docker.pkg.dev:443
       - name: Install and Run Integration Tests
         run: |
-          JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) \
-          && ./mvnw -B -PKubernetes,${{ matrix.suite }} clean verify -Djkube.version="$JKUBE_VERSION"
-      - name: Save reports as artifact
-        if: always()
-        uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05
-        with:
-          name: Test reports (Minikube ${{ matrix.kubernetes }}-${{ matrix.suite }})
-          path: ./it/target/jkube-test-report.txt
-
-  minikube-legacy:
-    name: K8S
-    needs: build-jkube
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        suite: ['quarkus','quarkus-native','springboot','webapp','other','dockerfile']
-    steps:
-      - name: Checkout
-        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f
-      - name: Setup Java 11
-        uses: actions/setup-java@5ffc13f4174014e2d4d4572b3d74c3fa61aeb2c2
-        with:
-          java-version: '11'
-          distribution: 'temurin'
-      - name: Cache configuration
-        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
-        with:
-          path: |
-            ~/.m2/repository
-            ./jkube
-          key: cache-it-${{ github.run_id }}
-      - name: Setup Minikube-Kubernetes
-        uses: manusa/actions-setup-minikube@1e9b544a8e64993bad1693f0aebeb108b459edd0
-        with:
-          minikube version: v1.25.2
-          kubernetes version: v1.12.10
-          github token: ${{ secrets.GITHUB_TOKEN }}
-          start args: --force
-      - name: Harden Runner
-        uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a
-        with:
-          egress-policy: block
-          allowed-endpoints: >
-            api.github.com:443
-            auth.docker.io:443
-            azure.archive.ubuntu.com:80
-            cdn03.quay.io:443
-            gcr.io:443
-            github.com:443
-            jcenter.bintray.com:443
-            k8s.gcr.io:443
-            maven.repository.redhat.com:443
-            md-hdd-51w5snc21ccf.z49.blob.storage.azure.net:443
-            md-hdd-bfh3mwcdlxsh.z21.blob.storage.azure.net:443
-            md-hdd-dxgvrxd2cnjf.z22.blob.storage.azure.net:443
-            objects.githubusercontent.com:443
-            packages.microsoft.com:443
-            plugins-artifacts.gradle.org:443
-            plugins.gradle.org:443
-            ppa.launchpadcontent.net:443
-            ppa.launchpad.net:80
-            production.cloudflare.docker.com:443
-            quay.io:443
-            registry-1.docker.io:443
-            registry.access.redhat.com:443
-            registry.k8s.io:443
-            repo1.maven.org:443
-            repo.maven.apache.org:443
-            repository.jboss.org:443
-            storage.googleapis.com:443
-            us-south1-docker.pkg.dev:443
-            us-west2-docker.pkg.dev:443
-            us-east4-docker.pkg.dev:443
-      - name: Install and Run Integration Tests
-        run: |
-          JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) \
+          eval $(minikube docker-env)                                                                                                          \
+          && JKUBE_VERSION=$(./mvnw -q -f 'jkube/pom.xml' -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)   \
           && ./mvnw -B -PKubernetes,${{ matrix.suite }} clean verify -Djkube.version="$JKUBE_VERSION"
       - name: Save reports as artifact
         if: always()
@@ -234,7 +159,7 @@ jobs:
         openshift: [v3.11.0,v3.10.0]
         suite: ['quarkus','springboot','webapp','other']
     steps:
-#     This seems to cause problems with OpenShift Setup Action 
+#     This seems to cause problems with OpenShift Setup Action
 #     - name: Harden Runner
 #       uses: step-security/harden-runner@8e0b4153524c217e5dce81df0986478e95a7004a
 #       with:
@@ -275,7 +200,7 @@ jobs:
             ./jkube
           key: cache-it-${{ github.run_id }}
       - name: Check Docker Status
-        run: systemctl status docker.service    
+        run: systemctl status docker.service
       - name: Setup OpenShift
         uses: manusa/actions-setup-openshift@4fbc3dc0710ea1f9ff4c7e1e6196dafac8e78a28
         with:

README.md

@@ -2,6 +2,16 @@
 
 This project hosts Integration test suites for https://github.com/eclipse/jkube.
 
+## Running locally
+
+### Minikube
+
+To be able to run the full suite in Minikube, you need to enable insecure registries (k8s:push tests).
+
+```shell
+minikube start --insecure-registry "0.0.0.0/0"
+```
+
 ## Test Structure
 
 In order to be able to run the tests in a CI environment without hogging the resources

it/pom.xml

@@ -44,6 +44,10 @@
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kubernetes-junit-jupiter</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.fabric8</groupId>
       <artifactId>openshift-client</artifactId>
@@ -85,11 +89,6 @@
       <artifactId>maven-invoker</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>io.fabric8</groupId>
-      <artifactId>kubernetes-junit-jupiter</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
   <build>
     <plugins>

it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/GradleExtension.java

@@ -13,6 +13,7 @@
  */
 package org.eclipse.jkube.integrationtests.jupiter.api.extension;
 
+import io.fabric8.junit.jupiter.BaseExtension;
 import org.eclipse.jkube.integrationtests.cli.CliUtils;
 import org.eclipse.jkube.integrationtests.gradle.JKubeGradleRunner;
 import org.eclipse.jkube.integrationtests.jupiter.api.Gradle;

it/src/main/java/org/eclipse/jkube/integrationtests/jupiter/api/extension/RegistryExtension.java

@@ -13,10 +13,12 @@
  */
 package org.eclipse.jkube.integrationtests.jupiter.api.extension;
 
+import io.fabric8.junit.jupiter.HasKubernetesClient;
+import io.fabric8.kubernetes.api.model.IntOrString;
+import io.fabric8.kubernetes.api.model.ServiceBuilder;
 import org.eclipse.jkube.integrationtests.cli.CliUtils;
 import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistry;
 import org.eclipse.jkube.integrationtests.jupiter.api.DockerRegistryHost;
-import org.hamcrest.Matchers;
 import org.junit.jupiter.api.extension.AfterAllCallback;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.BeforeEachCallback;
@@ -27,11 +29,14 @@
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
+import java.util.UUID;
+import java.util.stream.Stream;
 
 import static org.eclipse.jkube.integrationtests.cli.CliUtils.isWindows;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.notNullValue;
 
-public class RegistryExtension implements BaseExtension, BeforeAllCallback, BeforeEachCallback, AfterAllCallback {
+public class RegistryExtension implements HasKubernetesClient, BeforeAllCallback, BeforeEachCallback, AfterAllCallback {
 
   private static final Logger log = LoggerFactory.getLogger(RegistryExtension.class);
 
@@ -40,22 +45,23 @@ public void beforeAll(ExtensionContext context) throws Exception {
     final var annotation = context.getRequiredTestClass().getAnnotation(DockerRegistry.class);
     CliUtils.runCommand("docker rm -f " + getName(annotation));
     log.debug(() -> "Starting Docker Registry Extension");
-    final CliUtils.CliResult dockerRegistry;
+    final RegistryInfo dockerRegistry;
     if (isWindows()) {
       dockerRegistry = startWindowsDockerRegistry(annotation);
     } else {
-      dockerRegistry = startRegularDockerRegistry(annotation);
+      dockerRegistry = startKubernetesDockerRegistry(context);
     }
-    assertThat(dockerRegistry.getOutput(), dockerRegistry.getExitCode(), Matchers.equalTo(0));
+    assertThat(dockerRegistry.assertionContext, dockerRegistry.host, notNullValue());
+    getStore(context).put(RegistryInfo.class, dockerRegistry);
     log.debug(() -> "Docker Registry started successfully");
   }
 
   @Override
   public void beforeEach(ExtensionContext context) throws Exception {
-    final var annotation = context.getRequiredTestClass().getAnnotation(DockerRegistry.class);
+    final var registryInfo = getStore(context).get(RegistryInfo.class, RegistryInfo.class);
     for (Field f : context.getRequiredTestClass().getDeclaredFields()) {
       if (f.isAnnotationPresent(DockerRegistryHost.class) && f.getType() == String.class) {
-        setFieldValue(f, context.getRequiredTestInstance(), getDockerHost() + ":" + annotation.port());
+        setFieldValue(f, context.getRequiredTestInstance(), registryInfo.host + ":" + registryInfo.port);
       }
     }
   }
@@ -64,22 +70,45 @@ public void beforeEach(ExtensionContext context) throws Exception {
   public void afterAll(ExtensionContext context) throws Exception {
     log.debug(() -> "Closing Docker Registry");
     CliUtils.runCommand("docker stop " + getName(context.getRequiredTestClass().getAnnotation(DockerRegistry.class)));
+    if (!isWindows()) {
+      final var registryInfo = getStore(context).get(RegistryInfo.class, RegistryInfo.class);
+      final var client = getClient(context);
+      Stream.of(
+        client.pods().withName(registryInfo.name),
+        client.services().withName(registryInfo.name)
+      ).forEach(r -> r.withGracePeriod(0L).delete());
+    }
   }
 
-  private static CliUtils.CliResult startRegularDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException {
-    log.debug(() -> "Starting standard Docker Registry");
-    return CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " +
-      getName(dockerRegistry) + " registry:2");
+  private RegistryInfo startKubernetesDockerRegistry(ExtensionContext context) throws IOException, InterruptedException {
+    final var name = "registry" + UUID.randomUUID().toString().replace("-", "");
+    final var client = getClient(context);
+    final var ip = CliUtils.runCommand("minikube ip").getOutput().trim();
+    final var dockerRegistry = client.run().withName(name).withImage("registry:2")
+      .withNewRunConfig()
+      .addToLabels("app", "docker-registry").addToLabels("group", "jkube-it").done();
+    final var service = client.services().resource(new ServiceBuilder()
+      .withNewMetadata().withName(name)
+      .addToLabels("app", "docker-registry").addToLabels("group", "jkube-it").endMetadata()
+      .withNewSpec().withType("NodePort").withSelector(dockerRegistry.getMetadata().getLabels())
+      .addNewPort().withName("http").withPort(5000).withTargetPort(new IntOrString(5000)).endPort().endSpec()
+      .build())
+      .serverSideApply(); // Unsupported in K8s 1.12
+    return new RegistryInfo(name, ip, service.getSpec().getPorts().get(0).getNodePort(), null);
   }
 
-  private static CliUtils.CliResult startWindowsDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException {
+  private static RegistryInfo startWindowsDockerRegistry(DockerRegistry dockerRegistry) throws IOException, InterruptedException {
     log.debug(() -> "Starting Windows specific Docker Registry");
     final var registry = new File("C:\\registry");
     if (!registry.exists() && !registry.mkdirs()) {
       throw new IllegalStateException("Directory C:\\registry cannot be created");
     }
-    return CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " +
+    final var result = CliUtils.runCommand("docker run --rm -d -p " + dockerRegistry.port() +":5000 --name " +
       getName(dockerRegistry) + " -v C:\\registry:C:\\registry marcnuri/docker-registry-windows:ltsc2022");
+    if (result.getExitCode() != 0) {
+      return new RegistryInfo(null, null, -1, result.getOutput());
+    }
+    return new RegistryInfo("windows-docker-registry", getDockerHost(), dockerRegistry.port(), result.getOutput());
   }
 
   private static String getName(DockerRegistry dockerRegistry) {
@@ -95,4 +124,18 @@ private static String getDockerHost() {
         .replaceAll(":\\d+$", "");
     }
   }
+
+  private static final class RegistryInfo {
+    private final String name;
+    private final String host;
+    private final int port;
+    private final String assertionContext;
+
+    public RegistryInfo(String name, String host, int port, String assertionContext) {
+      this.name = name;
+      this.host = host;
+      this.port = port;
+      this.assertionContext = assertionContext;
+    }
+  }
 }