Merge branch 'master' into Qlik-Connector-Integration

.github/workflows/docker-unified.yml

@@ -64,6 +64,18 @@ jobs:
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
       - name: Compute Tag
         id: tag
         run: |

datahub-frontend/build.gradle

@@ -87,7 +87,7 @@ docker {
   }
 }
 
-task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
+task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', dist]) {
   from ("${buildDir}/distributions")
   include "datahub-frontend-${version}.zip"
   into "${buildDir}/docker/"

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/nocode/DataMigrationStep.java

@@ -10,6 +10,7 @@
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV1;
 import com.linkedin.metadata.entity.ebean.EbeanAspectV2;
@@ -170,7 +171,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
             // Emit a browse path aspect.
             final BrowsePaths browsePaths;
             try {
-              browsePaths = _entityService.buildDefaultBrowsePath(urn);
+              browsePaths = DefaultAspectsUtil.buildDefaultBrowsePath(urn, _entityService);
 
               final AuditStamp browsePathsStamp = new AuditStamp();
               browsePathsStamp.setActor(Urn.createFromString(Constants.SYSTEM_ACTOR));

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java

@@ -15,6 +15,7 @@
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.aspect.utils.DefaultAspectsUtil;
 import com.linkedin.metadata.entity.EntityService;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
@@ -181,7 +182,8 @@ private Filter backfillDefaultBrowsePathsV2Filter() {
   }
 
   private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception {
-    BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true);
+    BrowsePathsV2 browsePathsV2 =
+        DefaultAspectsUtil.buildDefaultBrowsePathV2(urn, true, _entityService);
     log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2));
     MetadataChangeProposal proposal = new MetadataChangeProposal();
     proposal.setEntityUrn(urn);

docker/profiles/docker-compose.prerequisites.yml

@@ -117,10 +117,10 @@ services:
     restart: on-failure
     healthcheck:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
-      start_period: 10s
-      interval: 1s
-      retries: 3
-      timeout: 5s
+      start_period: 20s
+      interval: 2s
+      timeout: 10s
+      retries: 5
     volumes:
       - ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
       - mysqldata:/var/lib/mysql

docs/cli.md

@@ -180,7 +180,7 @@ failure_log:
 ### init
 
 The init command is used to tell `datahub` about where your DataHub instance is located. The CLI will point to localhost DataHub by default.
-Running `datahub init` will allow you to customize the datahub instance you are communicating with.
+Running `datahub init` will allow you to customize the datahub instance you are communicating with. It has an optional `--use-password` option which allows to initialise the config using username, password. We foresee this mainly being used by admins as majority of organisations will be using SSO and there won't be any passwords to use.
 
 **_Note_**: Provide your GMS instance's host when the prompt asks you for the DataHub host.
 

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java

@@ -3,6 +3,7 @@
 import com.linkedin.metadata.aspect.plugins.validation.AspectRetriever;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.util.Pair;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -17,14 +18,14 @@
  * SystemMetadata} and record/message created time
  */
 public interface AspectsBatch {
-  List<? extends BatchItem> getItems();
+  Collection<? extends BatchItem> getItems();
 
   /**
    * Returns MCP items. Can be patch, upsert, etc.
    *
    * @return batch items
    */
-  default List<? extends MCPBatchItem> getMCPItems() {
+  default Collection<? extends MCPBatchItem> getMCPItems() {
     return getItems().stream()
         .filter(item -> item instanceof MCPBatchItem)
         .map(item -> (MCPBatchItem) item)

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java

@@ -2,11 +2,13 @@
 
 import com.linkedin.common.AuditStamp;
 import com.linkedin.common.urn.Urn;
+import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.mxe.SystemMetadata;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public interface BatchItem {
   /**
@@ -63,4 +65,12 @@ default String getAspectName() {
    */
   @Nonnull
   AspectSpec getAspectSpec();
+
+  /**
+   * The aspect's record template. Null when patch
+   *
+   * @return record template if it exists
+   */
+  @Nullable
+  RecordTemplate getRecordTemplate();
 }

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/MCLBatchItem.java

@@ -26,7 +26,7 @@ default String getAspectName() {
     if (getMetadataChangeLog().getAspectName() != null) {
       return getMetadataChangeLog().getAspectName();
     } else {
-      return getAspect().schema().getName();
+      return getRecordTemplate().schema().getName();
     }
   }
 
@@ -40,10 +40,7 @@ default SystemMetadata getPreviousSystemMetadata() {
   }
 
   @Nullable
-  RecordTemplate getPreviousAspect();
-
-  @Nonnull
-  RecordTemplate getAspect();
+  RecordTemplate getPreviousRecordTemplate();
 
   @Override
   @Nonnull

entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/UpsertItem.java

@@ -11,8 +11,6 @@
  * related data stored along with the aspect
  */
 public abstract class UpsertItem extends MCPBatchItem {
-  public abstract RecordTemplate getAspect();
-
   public abstract SystemAspect toLatestEntityAspect();
 
   public abstract void validatePreCommit(