feat: test verifying CRUD project operations on KG API (#3255)

* feat: test verifying CRUD project operations on KG API

* chore: graph version updated

CHANGELOG.rst

@@ -1,11 +1,47 @@
 .. _changelog:
 
+0.38.0
+------
+
+Renku ``0.38.0`` improves the Knowledge Graph API, with a new Project Creation functionality and a Project Update enhancement.
+
+User-Facing Changes
+~~~~~~~~~~~~~~~~~~~
+
+**🌟 New Features**
+
+- 🖼️ **Knowledge Graph**: New `Project Create API <https://renkulab.io/swagger/?urls.primaryName=knowledge%20graph#/default/post_projects>`_
+  to create a project in GitLab and Knowledge Graph
+  (`#1635 <https://github.com/SwissDataScienceCenter/renku-graph/issues/1635>`_).
+
+**🐞 Bug Fixes**
+
+- **Knowledge Graph**: Improves quality of the results returned by the Cross-Entity Search API.
+- **Knowledge Graph**: The `Project Update API <https://renkulab.io/swagger/?urls.primaryName=knowledge%20graph#/default/patch_projects__namespace___projectName_>`_ to work for non-public projects
+  (`#1695 <https://github.com/SwissDataScienceCenter/renku-graph/pull/1695>`_).
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**Bug Fixes**
+
+- **Knowledge Graph**: Various issues preventing Grafana dashboards not working.
+  (`#1717 <https://github.com/SwissDataScienceCenter/renku-graph/pull/1717>`_)
+  (`#1719 <https://github.com/SwissDataScienceCenter/renku-graph/pull/1719>`_).
+
+Individual components
+~~~~~~~~~~~~~~~~~~~~~~
+
+- `renku-graph 2.42.0 <https://github.com/SwissDataScienceCenter/renku-graph/releases/tag/2.42.0>`_
+- `renku-graph 2.42.1 <https://github.com/SwissDataScienceCenter/renku-graph/releases/tag/2.42.1>`_
+
+
 0.37.0
 ------
 
-Renku ``0.37.0`` introduces a new feature to pause sessions and later resume them exactly where you left off. All of your work in progress, including files, data, and environment changes not saved to git, are resumed right as you left them. 
+Renku ``0.37.0`` introduces a new feature to pause sessions and later resume them exactly where you left off. All of your work in progress, including files, data, and environment changes not saved to git, are resumed right as you left them.
 
-This feature replaces RenkuLab's branch-based auto-save mechanism. Most users do not have to do anything to transition from auto-saves to persistent sessions. However, if your last session went into an auto-save, you can still retrieve that work by using Start with Options and selecting your most recent auto-save branch. If your project contains auto-save branches that you do not need anymore, you can safely delete them. 
+This feature replaces RenkuLab's branch-based auto-save mechanism. Most users do not have to do anything to transition from auto-saves to persistent sessions. However, if your last session went into an auto-save, you can still retrieve that work by using Start with Options and selecting your most recent auto-save branch. If your project contains auto-save branches that you do not need anymore, you can safely delete them.
 
 User-Facing Changes
 ~~~~~~~~~~~~~~~~~~~

acceptance-tests/build.sbt

@@ -30,22 +30,23 @@ publishTo := Some(Resolver.file("Unused transient repository", file("target/unus
 
 val circeVersion = "0.14.6"
 
-libraryDependencies += "ch.qos.logback"          % "logback-classic"          % "1.4.11"
-libraryDependencies += "com.github.pureconfig"  %% "pureconfig"               % "0.17.4"     % Test
-libraryDependencies += "eu.timepit"             %% "refined"                  % "0.11.0"     % Test
-libraryDependencies += "io.circe"               %% "circe-core"               % circeVersion % Test
-libraryDependencies += "io.circe"               %% "circe-literal"            % circeVersion % Test
-libraryDependencies += "io.circe"               %% "circe-parser"             % circeVersion % Test
-libraryDependencies += "io.circe"               %% "circe-optics"             % "0.15.0"     % Test
-libraryDependencies += "org.http4s"             %% "http4s-blaze-client"      % "0.23.15"    % Test
-libraryDependencies += "org.http4s"             %% "http4s-circe"             % "0.23.23"    % Test
-libraryDependencies += "org.scalacheck"         %% "scalacheck"               % "1.17.0"     % Test
-libraryDependencies += "org.scalatest"          %% "scalatest"                % "3.2.17"     % Test
-libraryDependencies += "org.scalatestplus"      %% "selenium-4-1"             % "3.2.12.1"   % Test
-libraryDependencies += "org.seleniumhq.selenium" % "selenium-http-jdk-client" % "4.13.0"     % Test
-libraryDependencies += "org.seleniumhq.selenium" % "selenium-java"            % "4.7.1"      % Test
-libraryDependencies += "org.slf4j"               % "slf4j-log4j12"            % "2.0.9"      % Test
-libraryDependencies += "org.typelevel"          %% "cats-effect"              % "3.5.2"      % Test
+libraryDependencies += "ch.qos.logback"          % "logback-classic"               % "1.4.11"
+libraryDependencies += "com.github.pureconfig"  %% "pureconfig"                    % "0.17.4"     % Test
+libraryDependencies += "eu.timepit"             %% "refined"                       % "0.11.0"     % Test
+libraryDependencies += "io.circe"               %% "circe-core"                    % circeVersion % Test
+libraryDependencies += "io.circe"               %% "circe-literal"                 % circeVersion % Test
+libraryDependencies += "io.circe"               %% "circe-parser"                  % circeVersion % Test
+libraryDependencies += "io.circe"               %% "circe-optics"                  % "0.15.0"     % Test
+libraryDependencies += "org.http4s"             %% "http4s-blaze-client"           % "0.23.15"    % Test
+libraryDependencies += "org.http4s"             %% "http4s-circe"                  % "0.23.23"    % Test
+libraryDependencies += "org.scalacheck"         %% "scalacheck"                    % "1.17.0"     % Test
+libraryDependencies += "org.scalatest"          %% "scalatest"                     % "3.2.17"     % Test
+libraryDependencies += "org.scalatestplus"      %% "selenium-4-1"                  % "3.2.12.1"   % Test
+libraryDependencies += "org.seleniumhq.selenium" % "selenium-http-jdk-client"      % "4.13.0"     % Test
+libraryDependencies += "org.seleniumhq.selenium" % "selenium-java"                 % "4.7.1"      % Test
+libraryDependencies += "org.slf4j"               % "slf4j-log4j12"                 % "2.0.9"      % Test
+libraryDependencies += "org.typelevel"          %% "cats-effect"                   % "3.5.2"      % Test
+libraryDependencies += "org.typelevel"          %% "cats-effect-testing-scalatest" % "1.5.0"      % Test
 
 scalacOptions += "-feature"
 scalacOptions += "-unchecked"

acceptance-tests/src/test/scala/ch/renku/acceptancetests/BatchProjectRemovalSpec.scala

@@ -18,6 +18,7 @@
 
 package ch.renku.acceptancetests
 
+import ch.renku.acceptancetests.model.projects
 import ch.renku.acceptancetests.tooling.TestLogger.logger
 import ch.renku.acceptancetests.tooling.{AcceptanceSpec, KnowledgeGraphApi}
 import ch.renku.acceptancetests.workflows._
@@ -61,7 +62,7 @@ class BatchProjectRemovalSpec extends AcceptanceSpec with Login with RemoveProje
       case (summary, ProjectInfo(_, path, fullPath, created)) =>
         if (batchRemoveConfig.patterns.exists(_ matches path) && (created < Instant.now().minus(gracePeriod))) {
           logger.info(s"Removing '$path' - the removal pattern matched and it's older than ${gracePeriod.toDays} days")
-          `DELETE /knowledge-graph/projects/:slug`(fullPath)
+          `DELETE /knowledge-graph/projects/:slug`(projects.Slug(fullPath))
           summary.incrementRemoved()
         } else
           summary.incrementFound()

acceptance-tests/src/test/scala/ch/renku/acceptancetests/HandsOnSpec.scala

@@ -46,7 +46,7 @@ class HandsOnSpec
     val flightsDatasetName = `follow the flights tutorial`(projectUrl)
 
     When("all the events are processed by the knowledge-graph")
-    `wait for KG to process events`(projectDetails.asProjectIdentifier, webDriver)
+    `wait for KG to process events`(projectDetails.asProjectIdentifier.asProjectSlug, webDriver)
 
     `verify dataset was created`(flightsDatasetName)
 

acceptance-tests/src/test/scala/ch/renku/acceptancetests/ImportZenodoWithCliSpec.scala

@@ -69,7 +69,7 @@ class ImportZenodoWithCliSpec
     sleep(10 seconds)
 
     When("all the events are processed by the knowledge-graph")
-    `wait for KG to process events`(projectDetails.asProjectIdentifier, webDriver)
+    `wait for KG to process events`(projectDetails.asProjectIdentifier.asProjectSlug, webDriver)
 
     sleep(5 seconds)
 

acceptance-tests/src/test/scala/ch/renku/acceptancetests/ProjectAPISpec.scala

@@ -21,29 +21,55 @@ package ch.renku.acceptancetests
 import cats.syntax.all._
 import ch.renku.acceptancetests.generators.Generators.Implicits._
 import ch.renku.acceptancetests.model.projects
-import ch.renku.acceptancetests.tooling.{AcceptanceSpec, KnowledgeGraphApi}
-import ch.renku.acceptancetests.workflows.{Login, Project}
+import ch.renku.acceptancetests.tooling.{AcceptanceSpec, GitLabApi, KnowledgeGraphApi}
+import ch.renku.acceptancetests.workflows.Login
 import org.scalacheck.Gen
+import org.scalatest.OptionValues
 import tooling.KnowledgeGraphModel._
 
-class ProjectAPISpec extends AcceptanceSpec with Login with Project with KnowledgeGraphApi {
+import scala.concurrent.duration._
 
-  scenario("User can update project using the API") {
+class ProjectAPISpec extends AcceptanceSpec with Login with KnowledgeGraphApi with GitLabApi with OptionValues {
+
+  scenario("User can do CRUD operations on a project using the API") {
 
     `log in to Renku`
 
-    `create, continue or open a project`
+    When("the user creates a new Project")
+    val namespaceId = `find user namespace ids`.headOption.getOrElse(fail("No namespaces found"))
+    val newProject  = NewProject.generate(namespaceId, ProjectTemplate.pythonMinimal, Image.wheelPngExample)
+    val slug        = `POST /knowledge-graph/projects`(newProject)
+
+    `wait for KG to process events`(slug, webDriver)
+
+    Then("the user should be able to get details of it")
+    val afterCreation = `GET /knowledge-graph/projects/:slug`(slug).value
+    newProject.visibility                      shouldBe afterCreation.visibility
+    newProject.maybeDescription                shouldBe afterCreation.maybeDescription
+    newProject.keywords                        shouldBe afterCreation.keywords
+    newProject.maybeImage.map(_.toName).toList shouldBe afterCreation.images.map(_.toName)
 
     When("the user updates the project")
-    val newDesc       = Some("new description")
-    val newKeywords   = Set("new keyword")
-    val newVisibility = Gen.oneOf(projects.Visibility.all).suchThat(_ != projectDetails.visibility).generateOne
-    val updates =
-      ProjectUpdates(newDescription = newDesc.some, newKeywords = newKeywords.some, newVisibility = newVisibility.some)
-    `PATCH /knowledge-graph/projects/:slug`(projectDetails.asProjectSlug, updates)
+    val newDesc       = projects.Description.generate().some
+    val newKeywords   = Set(projects.Keyword.generate())
+    val newVisibility = Gen.oneOf(projects.Visibility.all).suchThat(_ != newProject.visibility).generateOne
+    val newImage      = Image.bikeJpgExample.some
+    val updates       = ProjectUpdates(newDesc.some, newKeywords.some, newVisibility.some, newImage.some)
+    `PATCH /knowledge-graph/projects/:slug`(slug, updates)
+
+    `wait for KG to process events`(slug, webDriver)
 
     Then("the API should return the updated values")
-    `GET /knowledge-graph/projects/:slug`(projectDetails.asProjectSlug) shouldBe
-      KGProjectDetails(description = newDesc, keywords = newKeywords, visibility = newVisibility)
+    val afterUpdate = `GET /knowledge-graph/projects/:slug`(slug).value
+    newVisibility                 shouldBe afterUpdate.visibility
+    newDesc                       shouldBe afterUpdate.maybeDescription
+    newKeywords                   shouldBe afterUpdate.keywords
+    newImage.map(_.toName).toList shouldBe afterUpdate.images.map(_.toName)
+
+    When("the user removes the project")
+    `DELETE /knowledge-graph/projects/:slug`(slug) sleep (5 seconds)
+
+    Then("the API should not be able to find the project any more")
+    `GET /knowledge-graph/projects/:slug`(slug) shouldBe None
   }
 }

acceptance-tests/src/test/scala/ch/renku/acceptancetests/generators/Generators.scala

@@ -93,9 +93,9 @@ object Generators {
       lines <- Gen.listOfN(size, nonEmptyStrings())
     } yield lines
 
-  def listOf[T](generator: Gen[T], maxElements: Int Refined Positive = 5): Gen[List[T]] =
+  def listOf[T](generator: Gen[T], min: Int = 0, max: Int = 5): Gen[List[T]] =
     for {
-      size <- choose(0, maxElements.value)
+      size <- choose(min, max)
       list <- Gen.listOfN(size, generator)
     } yield list
 
@@ -184,14 +184,28 @@ object Generators {
 
     implicit class GenOps[T](generator: Gen[T]) {
 
-      def generateOne: T = generator.sample getOrElse generateOne
+      def generateOne: T = generateExample(generator)
+
+      def generateList(min: Int = 0, max: Int = 5): List[T] =
+        generateExample(listOf(generator, min, max))
 
       def generateDifferentThan(value: T): T = {
         val generated = generator.sample.getOrElse(generateDifferentThan(value))
         if (generated == value) generateDifferentThan(value)
         else generated
       }
 
+      protected def generateExample[O](generator: Gen[O]): O = {
+        @annotation.tailrec
+        def loop(tries: Int): O =
+          generator.sample match {
+            case Some(o)               => o
+            case None if tries >= 5000 => sys.error(s"Failed to generate example value after $tries tries")
+            case None                  => loop(tries + 1)
+          }
+
+        loop(0)
+      }
     }
 
     implicit def asArbitrary[T](implicit generator: Gen[T]): Arbitrary[T] = Arbitrary(generator)

acceptance-tests/src/test/scala/ch/renku/acceptancetests/model/images.scala

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 Swiss Data Science Center (SDSC)
+ * A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+ * Eidgenössische Technische Hochschule Zürich (ETHZ).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.renku.acceptancetests.model
+
+import ch.renku.acceptancetests.generators.Generators.Implicits._
+import ch.renku.acceptancetests.generators.Generators.httpUrls
+import org.scalacheck.Gen
+
+object images {
+
+  final case class Name(value: String) { override lazy val toString: String = value }
+
+  final case class ImageUri(value: String) {
+    override lazy val toString: String = value
+    lazy val toName:            Name   = Name(value.split("/").last)
+  }
+  object ImageUri {
+    val generator:  Gen[ImageUri] = httpUrls.map(ImageUri(_))
+    def generate(): ImageUri      = generator.generateOne
+  }
+}

acceptance-tests/src/test/scala/ch/renku/acceptancetests/model/projects.scala

@@ -24,9 +24,11 @@ import ch.renku.acceptancetests.generators.Generators._
 import ch.renku.acceptancetests.model.AuthorizationToken.{OAuthAccessToken, PersonalAccessToken}
 import ch.renku.acceptancetests.model.projects.ProjectDetails._
 import ch.renku.acceptancetests.model.users.UserCredentials
+import ch.renku.acceptancetests.tooling.multipart.PartValueEncoder
 import io.circe.DecodingFailure.Reason
 import io.circe.syntax._
 import io.circe.{Decoder, DecodingFailure, Encoder}
+import org.scalacheck.Gen
 
 import java.net.URI
 import java.time.LocalDateTime
@@ -36,8 +38,8 @@ import scala.util.Random
 object projects {
 
   final case class ProjectIdentifier(namespace: String, path: String) {
-    lazy val asProjectSlug:     String = s"$namespace/$path"
-    override lazy val toString: String = s"$namespace/$path"
+    lazy val asProjectSlug:     projects.Slug = projects.Slug(s"$namespace/$path")
+    override lazy val toString: String        = s"$namespace/$path"
   }
 
   final case class ProjectDetails(
@@ -53,20 +55,70 @@ object projects {
         path = title.toPathSegment
       )
 
-    def asProjectSlug(implicit userCredentials: UserCredentials): String =
+    def asProjectSlug(implicit userCredentials: UserCredentials): projects.Slug =
       asProjectIdentifier.asProjectSlug
   }
 
-  sealed abstract class Visibility(val value: String)
+  final case class Slug(value: String) { override lazy val toString: String = value }
+
+  final case class Name(value: String) { override lazy val toString: String = value }
+  object Name {
+
+    implicit val pvEncoder: PartValueEncoder[Name] = _.value
+
+    def generate(): Name = Name(
+      s"test ${LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss"))} ${Random.nextInt(100)}"
+    )
+  }
+
+  final case class NamespaceId(value: Int) { override lazy val toString: String = value.toString }
+  object NamespaceId {
+    implicit val pvEncoder: PartValueEncoder[NamespaceId] = _.value.toString
+  }
+
+  final case class Description(value: String) { override lazy val toString: String = value }
+  object Description {
+    def generate(): Description = nonEmptyStrings().map(Description(_)).generateOne
+
+    implicit val pvEncoder:   PartValueEncoder[Description] = _.value
+    implicit val jsonEncoder: Encoder[Description]          = Encoder.instance(_.value.asJson)
+    implicit val jsonDecoder: Decoder[Description]          = Decoder.instance(_.as[String].map(Description(_)))
+  }
+
+  final case class Keyword(value: String) { override lazy val toString: String = value }
+  object Keyword {
+    val generator:  Gen[Keyword] = nonEmptyStrings().map(Keyword(_))
+    def generate(): Keyword      = generator.generateOne
+
+    implicit val pvEncoder:   PartValueEncoder[Keyword] = _.value
+    implicit val jsonEncoder: Encoder[Keyword]          = Encoder.instance(_.value.asJson)
+    implicit val jsonDecoder: Decoder[Keyword]          = Decoder.instance(_.as[String].map(Keyword(_)))
+  }
 
+  final case class TemplateRepoUrl(value: String) { override lazy val toString: String = value }
+  object TemplateRepoUrl {
+    implicit val pvEncoder: PartValueEncoder[TemplateRepoUrl] = _.value
+  }
+
+  final case class TemplateId(value: String) { override lazy val toString: String = value }
+  object TemplateId {
+    implicit val pvEncoder: PartValueEncoder[TemplateId] = _.value
+  }
+
+  final case class Template(name: String) { override lazy val toString: String = name }
+
+  sealed abstract class Visibility(val value: String)
   object Visibility {
     case object Public   extends Visibility(value = "public")
     case object Private  extends Visibility(value = "private")
     case object Internal extends Visibility(value = "internal")
 
     val all: Set[Visibility] = Set(Public, Internal, Private)
 
-    implicit val jsonEncoder: Encoder[Visibility] = Encoder.instance(_.value.asJson)
+    def generate(): Visibility = Gen.oneOf(all).generateOne
+
+    implicit val pvEncoder:   PartValueEncoder[Visibility] = _.value
+    implicit val jsonEncoder: Encoder[Visibility]          = Encoder.instance(_.value.asJson)
     implicit val jsonDecoder: Decoder[Visibility] = Decoder.instance { cur =>
       cur.as[String].flatMap { v =>
         all
@@ -78,14 +130,7 @@ object projects {
     }
   }
 
-  case class Template(name: String) {
-    override lazy val toString: String = name
-  }
-
-  final case class ProjectUrl(value: String) {
-    override lazy val toString: String = value
-  }
-
+  final case class ProjectUrl(value: String) { override lazy val toString: String = value }
   object ProjectUrl {
 
     implicit class ProjectUrlOps(projectUrl: ProjectUrl)(implicit userCredentials: UserCredentials) {
@@ -113,10 +158,7 @@ object projects {
                  maybeDescription: Option[String] = None,
                  maybeTemplate:    Option[String] = None
     ): ProjectDetails = {
-      val now = LocalDateTime.now()
-      val title = maybeTitle.getOrElse(
-        s"test ${now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss"))} ${Random.nextInt(100)}"
-      )
+      val title = maybeTitle.getOrElse(Name.generate().value)
       val desc = maybeDescription.getOrElse(
         prefixParagraph("Generated by tests: ", maxWords = 3).generateOne
       )