feat: add ephemeral kubernetes provider (#4927)

* chore: wip

* chore: wip2

* chore: wip3

* chore: improve logs

* chore: remove dockerhub imagepull secret

* chore: add ephemeral ingresses mvp

* chore: add service ports annotations

* chore: add cluster deadline and api response types

* fix: lint

* chore: wip

* chore: add ephemeral ingress chart

* chore: update secret name

* refactor: remove unnecessary config and use api types from pkg

* chore: fix ephemeral-nginx config

* chore: remove old changes that are not needed anymore

* docs: add docs for ephemeral-clusters

* chore: update error class

* chore: fix lint

* fix:  lint

* test: add some tests for provider

* docs: add an example project using ephemeral-kubernetes provider

* chore: change error type to configuration error

* docs: link example in docs

* chore: show message returned from the api in case of api error

* docs: typo

* chore: set protocol as https for ephemeral cluster ingresses

* docs: update docs

* docs: update sidebar title

* chore: update docs

* chore: fix lint

* chore: address comments

* chore: rebase and regenerate docs

* chore: fix lint

* chore: remove unused imports

* chore: remove link temporarily to avoid dead link

* chore: update docs

core/package.json

@@ -165,7 +165,7 @@
   "devDependencies": {
     "@commitlint/cli": "^17.6.5",
     "@commitlint/config-conventional": "^17.6.5",
-    "@garden-io/platform-api-types": "1.455.0",
+    "@garden-io/platform-api-types": "1.914.0",
     "@google-cloud/kms": "^3.7.0",
     "@types/analytics-node": "^3.1.10",
     "@types/async": "^3.2.18",

core/src/cloud/api.ts

@@ -16,11 +16,14 @@ import { Cookie } from "tough-cookie"
 import { cloneDeep, isObject } from "lodash"
 import { dedent, deline } from "../util/string"
 import {
-  GetProjectResponse,
-  GetProfileResponse,
+  BaseResponse,
+  CreateEphemeralClusterResponse,
   CreateProjectsForRepoResponse,
+  EphemeralClusterWithRegistry,
+  GetKubeconfigResponse,
+  GetProfileResponse,
+  GetProjectResponse,
   ListProjectsResponse,
-  BaseResponse,
 } from "@garden-io/platform-api-types"
 import { getCloudDistributionName, getCloudLogSectionName, getPackageVersion } from "../util/util"
 import { CommandInfo } from "../plugin-context"
@@ -36,6 +39,10 @@ const gardenClientVersion = getPackageVersion()
 export class CloudApiDuplicateProjectsError extends CloudApiError {}
 export class CloudApiTokenRefreshError extends CloudApiError {}
 
+function extractErrorMessageBodyFromGotError(error: any): error is GotHttpError {
+  return error?.response?.body?.message
+}
+
 function stripLeadingSlash(str: string) {
   return str.replace(/^\/+/, "")
 }
@@ -786,4 +793,28 @@ export class CloudApi {
     }
     return secrets
   }
+
+  async createEphemeralCluster(): Promise<EphemeralClusterWithRegistry> {
+    try {
+      const response = await this.post<CreateEphemeralClusterResponse>(`/ephemeral-clusters/`)
+      return response.data
+    } catch (err) {
+      throw new CloudApiError({
+        message: `${extractErrorMessageBodyFromGotError(err) ?? "Creating an ephemeral cluster failed."}`,
+      })
+    }
+  }
+
+  async getKubeConfigForCluster(clusterId: string): Promise<string> {
+    try {
+      const response = await this.get<GetKubeconfigResponse>(`/ephemeral-clusters/${clusterId}/kubeconfig`)
+      return response.data.kubeconfig
+    } catch (err) {
+      throw new CloudApiError({
+        message: `${
+          extractErrorMessageBodyFromGotError(err) ?? "Fetching the Kubeconfig for ephemeral cluster failed."
+        }`,
+      })
+    }
+  }
 }

core/src/plugins/kubernetes/container/ingress.ts

@@ -20,6 +20,7 @@ import { V1Ingress, V1Secret } from "@kubernetes/client-node"
 import { Log } from "../../../logger/log-entry"
 import chalk from "chalk"
 import { Resolved } from "../../../actions/types"
+import { isProviderEphemeralKubernetes } from "../ephemeral/ephemeral"
 
 // Ingress API versions in descending order of preference
 export const supportedIngressApiVersions = ["networking.k8s.io/v1", "networking.k8s.io/v1beta1", "extensions/v1beta1"]
@@ -183,8 +184,14 @@ async function getIngress(
 
   const certificate = await pickCertificate(action, api, provider, hostname)
   // TODO: support other protocols
-  const protocol: ServiceProtocol = !!certificate ? "https" : "http"
-  const port = !!certificate ? provider.config.ingressHttpsPort : provider.config.ingressHttpPort
+  let protocol: ServiceProtocol = !!certificate ? "https" : "http"
+  let port = !!certificate ? provider.config.ingressHttpsPort : provider.config.ingressHttpPort
+
+  // ephemeral-kubernetes ingresses should always be https
+  if (isProviderEphemeralKubernetes(provider)) {
+    protocol = "https"
+    port = provider.config.ingressHttpsPort
+  }
 
   return {
     ...spec,

core/src/plugins/kubernetes/ephemeral/config.ts

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import chalk from "chalk"
+import { mkdirp, writeFile } from "fs-extra"
+import { load } from "js-yaml"
+import { remove } from "lodash"
+import moment from "moment"
+import { join } from "path"
+import { joi, joiProviderName } from "../../../config/common"
+import { providerConfigBaseSchema } from "../../../config/provider"
+import { ConfigurationError } from "../../../exceptions"
+import { ConfigureProviderParams } from "../../../plugin/handlers/Provider/configureProvider"
+import { dedent } from "../../../util/string"
+import { KubernetesConfig, namespaceSchema } from "../config"
+import { EPHEMERAL_KUBERNETES_PROVIDER_NAME } from "./ephemeral"
+
+export const configSchema = () =>
+  providerConfigBaseSchema()
+    .keys({
+      name: joiProviderName(EPHEMERAL_KUBERNETES_PROVIDER_NAME),
+      namespace: namespaceSchema().description(
+        "Specify which namespace to deploy services to (defaults to the project name). " +
+          "Note that the framework generates other namespaces as well with this name as a prefix."
+      ),
+      setupIngressController: joi
+        .string()
+        .allow("nginx", false, null)
+        .default("nginx")
+        .description(
+          dedent`Set this to null or false to skip installing/enabling the \`nginx\` ingress controller. Note: if you skip installing the \`nginx\` ingress controller for ephemeral cluster, your ingresses may not function properly.`
+        ),
+    })
+    .description(`The provider configuration for the ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} plugin.`)
+
+export async function configureProvider(params: ConfigureProviderParams<KubernetesConfig>) {
+  const { base, log, projectName, ctx, config: baseConfig } = params
+  if (projectName === "garden-system") {
+    // avoid configuring ephemeral-kubernetes provider and creating ephemeral-cluster for garden-system project
+    return {
+      config: baseConfig,
+    }
+  }
+  log.info(`Configuring ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider for project ${projectName}`)
+  if (!ctx.cloudApi) {
+    throw new ConfigurationError({
+      message: `You are not logged in. You must be logged into Garden Cloud in order to use ${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider.`,
+    })
+  }
+  if (ctx.cloudApi && ctx.cloudApi?.domain !== "https://app.garden.io") {
+    throw new ConfigurationError({
+      message: `${EPHEMERAL_KUBERNETES_PROVIDER_NAME} provider is currently not supported for ${ctx.cloudApi.distroName}.`,
+    })
+  }
+  // creating tmp dir .garden/ephemeral-kubernetes for storing kubeconfig
+  const ephemeralClusterDirPath = join(ctx.gardenDirPath, "ephemeral-kubernetes")
+  await mkdirp(ephemeralClusterDirPath)
+  log.info("Creating ephemeral kubernetes cluster")
+  const createEphemeralClusterResponse = await ctx.cloudApi.createEphemeralCluster()
+  const clusterId = createEphemeralClusterResponse.instanceMetadata.instanceId
+  log.info(`Ephemeral kubernetes cluster created successfully`)
+  const deadlineDateTime = moment(createEphemeralClusterResponse.instanceMetadata.deadline)
+  const diffInNowAndDeadline = moment.duration(deadlineDateTime.diff(moment())).asMinutes().toFixed(1)
+  log.info(
+    chalk.white(
+      `Ephemeral cluster will be destroyed in ${diffInNowAndDeadline} minutes, at ${deadlineDateTime.format(
+        "YYYY-MM-DD HH:mm:ss"
+      )}`
+    )
+  )
+  log.info("Getting Kubeconfig for the cluster")
+  const kubeConfig = await ctx.cloudApi.getKubeConfigForCluster(clusterId)
+  const kubeconfigFileName = `${clusterId}-kubeconfig.yaml`
+  const kubeConfigPath = join(ctx.gardenDirPath, "ephemeral-kubernetes", kubeconfigFileName)
+  await writeFile(kubeConfigPath, kubeConfig)
+  log.info(`Kubeconfig for ephemeral cluster saved at path: ${chalk.underline(kubeConfigPath)}`)
+
+  const parsedKubeConfig: any = load(kubeConfig)
+  const currentContext = parsedKubeConfig["current-context"]
+  baseConfig.context = currentContext
+  baseConfig.kubeconfig = kubeConfigPath
+
+  // set deployment registry
+  baseConfig.deploymentRegistry = {
+    hostname: createEphemeralClusterResponse.registry.endpointAddress,
+    namespace: createEphemeralClusterResponse.registry.repository,
+    insecure: false,
+  }
+  // set imagePullSecrets
+  baseConfig.imagePullSecrets = [
+    {
+      name: createEphemeralClusterResponse.registry.imagePullSecret.name,
+      namespace: createEphemeralClusterResponse.registry.imagePullSecret.namespace,
+    },
+  ]
+  // set build mode to kaniko
+  baseConfig.buildMode = "kaniko"
+  // set additional kaniko flags
+  baseConfig.kaniko = {
+    extraFlags: [
+      `--registry-mirror=${createEphemeralClusterResponse.registry.endpointAddress}`,
+      `--registry-mirror=${createEphemeralClusterResponse.registry.dockerRegistryMirror}`,
+      "--insecure-pull",
+      "--force",
+    ],
+  }
+  // set setupIngressController to null while initializing kubernetes plugin
+  // as we use it later and configure it separately for ephemeral-kubernetes
+  const kubernetesPluginConfig = {
+    ...params,
+    config: {
+      ...baseConfig,
+      setupIngressController: null,
+    },
+  }
+  let { config: updatedConfig } = await base!(kubernetesPluginConfig)
+
+  // setup ingress controller unless setupIngressController is set to false/null in provider config
+  if (baseConfig.setupIngressController) {
+    const _systemServices = updatedConfig._systemServices
+    const nginxServices = ["ingress-controller", "default-backend"]
+    remove(_systemServices, (s) => nginxServices.includes(s))
+    _systemServices.push("nginx-ephemeral")
+    updatedConfig.setupIngressController = "nginx"
+    // set default hostname
+    updatedConfig.defaultHostname = createEphemeralClusterResponse.ingressesHostname
+  }
+
+  return {
+    config: updatedConfig,
+  }
+}

core/src/plugins/kubernetes/ephemeral/ephemeral.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <[email protected]>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+import { configureProvider, configSchema } from "./config"
+import { createGardenPlugin } from "../../../plugin/plugin"
+import { dedent } from "../../../util/string"
+import { KubernetesProvider } from "../config"
+import { joi, joiIdentifier } from "../../../config/common"
+
+const providerUrl = "./kubernetes.md"
+export const EPHEMERAL_KUBERNETES_PROVIDER_NAME = "ephemeral-kubernetes"
+
+const outputsSchema = joi.object().keys({
+  "app-namespace": joiIdentifier().required().description("The primary namespace used for resource deployments."),
+  "default-hostname": joi
+    .string()
+    .description(
+      "The dynamic hostname assigned to the ephemeral cluster automatically, when an ephemeral cluster is created."
+    ),
+})
+
+export const gardenPlugin = () =>
+  createGardenPlugin({
+    name: EPHEMERAL_KUBERNETES_PROVIDER_NAME,
+    base: "kubernetes",
+    docs: dedent`
+    The \`${EPHEMERAL_KUBERNETES_PROVIDER_NAME}\` provider is a specialized version of the [\`kubernetes\` provider](${providerUrl}) that allows to deploy applications to one of the ephemeral Kubernetes clusters provided by Garden.
+
+    For information about using ephemeral Kubernetes clusters, please refer to [Ephemeral Kubernetes clusters guide](../../basics/ephemeral-clusters.md)
+  `,
+    configSchema: configSchema(),
+    outputsSchema,
+    handlers: {
+      configureProvider,
+    },
+  })
+
+export function isProviderEphemeralKubernetes(provider: KubernetesProvider) {
+  return provider?.name === EPHEMERAL_KUBERNETES_PROVIDER_NAME
+}

core/src/plugins/kubernetes/helm/deployment.ts

@@ -167,7 +167,7 @@ export const helmDeploy: DeployActionHandler<"deploy", HelmDeployAction> = async
     attached = true
   }
   // Get ingresses of deployed resources
-  const ingresses = getK8sIngresses(manifests)
+  const ingresses = getK8sIngresses(manifests, provider)
 
   return {
     state: "ready",

core/src/plugins/kubernetes/helm/status.ts

@@ -72,7 +72,7 @@ export const getHelmDeployStatus: DeployActionHandler<"getStatus", HelmDeployAct
     const deployedResources = await getDeployedChartResources({ ctx: k8sCtx, action, releaseName, log })
 
     forwardablePorts = getForwardablePorts({ resources: deployedResources, parentAction: action, mode: deployedMode })
-    ingresses = getK8sIngresses(deployedResources)
+    ingresses = getK8sIngresses(deployedResources, provider)
 
     if (state === "ready") {
       // Local mode always takes precedence over sync mode

core/src/plugins/kubernetes/init.ts

@@ -35,6 +35,7 @@ import { mapValues, omit } from "lodash"
 import { getIngressApiVersion, supportedIngressApiVersions } from "./container/ingress"
 import { Log } from "../../logger/log-entry"
 import { DeployStatusMap } from "../../plugin/handlers/Deploy/get-status"
+import { isProviderEphemeralKubernetes } from "./ephemeral/ephemeral"
 
 const dockerAuthSecretType = "kubernetes.io/dockerconfigjson"
 const dockerAuthDocsLink = `
@@ -230,9 +231,9 @@ export async function prepareSystem({
     return {}
   }
 
-  // We require manual init if we're installing any system services to remote clusters, to avoid conflicts
-  // between users or unnecessary work.
-  if (!clusterInit && remoteCluster) {
+  // We require manual init if we're installing any system services to remote clusters unless the remote cluster
+  // is an ephemeral-cluster, to avoid conflicts between users or unnecessary work.
+  if (!clusterInit && remoteCluster && !isProviderEphemeralKubernetes(provider)) {
     const initCommand = chalk.white.bold(`garden --env=${ctx.environmentName} plugins kubernetes cluster-init`)
 
     if (combinedState === "ready") {

core/src/plugins/kubernetes/kubernetes-type/handlers.ts

@@ -214,7 +214,7 @@ export const getKubernetesDeployStatus: DeployActionHandler<"getStatus", Kuberne
       version: state === "ready" ? action.versionString() : undefined,
       detail: { remoteResources },
       mode: deployedMode,
-      ingresses: getK8sIngresses(remoteResources),
+      ingresses: getK8sIngresses(remoteResources, provider),
     },
     // TODO-0.13.1
     outputs: {},

core/src/plugins/kubernetes/status/ingress.ts

@@ -6,15 +6,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { ServiceIngress } from "../../../types/service"
+import { ServiceIngress, ServiceProtocol } from "../../../types/service"
+import { KubernetesProvider } from "../config"
+import { isProviderEphemeralKubernetes } from "../ephemeral/ephemeral"
 import { KubernetesIngress, KubernetesResource } from "../types"
 
 /**
  * Returns a list of ServiceIngresses found in a list of k8s resources.
  *
  * Does a best-effort extraction based on known ingress resource types.
  */
-export function getK8sIngresses(resources: KubernetesResource[]): ServiceIngress[] {
+export function getK8sIngresses(resources: KubernetesResource[], provider?: KubernetesProvider): ServiceIngress[] {
   const output: ServiceIngress[] = []
 
   for (const r of resources.filter(isIngressResource)) {
@@ -40,9 +42,15 @@ export function getK8sIngresses(resources: KubernetesResource[]): ServiceIngress
           stringPath = path
         }
 
+        let protocol: ServiceProtocol = tlsHosts.includes(rule.host) ? "https" : "http"
+        // ephemeral-kubernetes ingresses should always be https
+        if (provider && isProviderEphemeralKubernetes(provider)) {
+          protocol = "https"
+        }
+
         output.push({
           hostname: rule.host,
-          protocol: tlsHosts.includes(rule.host) ? "https" : "http",
+          protocol,
           path: stringPath,
         })
       }

core/src/plugins/plugins.ts

@@ -13,6 +13,7 @@ export const getSupportedPlugins = () => [
   { name: "hadolint", callback: () => require("./hadolint/hadolint").gardenPlugin.getSpec() },
   { name: "kubernetes", callback: () => require("./kubernetes/kubernetes").gardenPlugin() },
   { name: "local-kubernetes", callback: () => require("./kubernetes/local/local").gardenPlugin() },
+  { name: "ephemeral-kubernetes", callback: () => require("./kubernetes/ephemeral/ephemeral").gardenPlugin() },
   { name: "openshift", callback: () => require("./openshift/openshift").gardenPlugin() },
   { name: "octant", callback: () => require("./octant/octant").gardenPlugin() },
   { name: "otel-collector", callback: () => require("./otel-collector/otel-collector").gardenPlugin.getSpec() },

core/test/helpers/api.ts

@@ -42,6 +42,8 @@ export class FakeCloudApi extends CloudApi {
       cachedPermissions: {},
       accessTokens: [],
       groups: [],
+      meta: {},
+      singleProjectId: "",
     }
   }