Merge branch 'master' into improve_iceberg_connector

datahub-frontend/app/auth/AuthModule.java

@@ -27,6 +27,7 @@
 import io.datahubproject.metadata.context.EntityRegistryContext;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
+import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.SearchContext;
 import io.datahubproject.metadata.context.ValidationContext;
 import java.nio.charset.StandardCharsets;
@@ -195,6 +196,7 @@ protected OperationContext provideOperationContext(
         .searchContext(SearchContext.EMPTY)
         .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
         .validationContext(ValidationContext.builder().alternateValidation(false).build())
+        .retrieverContext(RetrieverContext.EMPTY)
         .build(systemAuthentication);
   }
 

datahub-frontend/conf/logback.xml

@@ -61,7 +61,7 @@
   <logger name="com.linkedin" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
-  <logger name="controller" level="DEBUG">
+  <logger name="controllers" level="DEBUG">
     <appender-ref ref="DEBUG_FILE"/>
   </logger>
   <logger name="auth" level="DEBUG">

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java

@@ -13,6 +13,7 @@
 import com.linkedin.gms.factory.kafka.common.TopicConventionFactory;
 import com.linkedin.gms.factory.kafka.schemaregistry.InternalSchemaRegistryFactory;
 import com.linkedin.gms.factory.search.BaseElasticSearchComponentsFactory;
+import com.linkedin.metadata.aspect.CachingAspectRetriever;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;
 import com.linkedin.metadata.dao.producer.KafkaEventProducer;
 import com.linkedin.metadata.dao.producer.KafkaHealthChecker;
@@ -186,6 +187,7 @@ protected OperationContext javaSystemOperationContext(
             components.getIndexConvention(),
             RetrieverContext.builder()
                 .aspectRetriever(entityServiceAspectRetriever)
+                .cachingAspectRetriever(CachingAspectRetriever.EMPTY)
                 .graphRetriever(systemGraphRetriever)
                 .searchRetriever(searchServiceSearchRetriever)
                 .build(),

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/restorebackup/RestoreStorageStep.java

@@ -180,7 +180,7 @@ private void readerExecutable(ReaderWrapper reader, UpgradeContext context) {
       try {
         aspectRecord =
             EntityUtils.toSystemAspect(
-                    context.opContext().getRetrieverContext().get(), aspect.toEntityAspect())
+                    context.opContext().getRetrieverContext(), aspect.toEntityAspect())
                 .get()
                 .getRecordTemplate();
       } catch (Exception e) {

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java

@@ -113,8 +113,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
                   List<Pair<Future<?>, SystemAspect>> futures;
                   futures =
                       EntityUtils.toSystemAspectFromEbeanAspects(
-                              opContext.getRetrieverContext().get(),
-                              batch.collect(Collectors.toList()))
+                              opContext.getRetrieverContext(), batch.collect(Collectors.toList()))
                           .stream()
                           .map(
                               systemAspect -> {

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java

@@ -100,8 +100,8 @@ static AspectsBatch generateAspectBatch(
             .collect(Collectors.toList());
 
     return AspectsBatchImpl.builder()
-        .mcps(mcps, auditStamp, opContext.getRetrieverContext().get())
-        .retrieverContext(opContext.getRetrieverContext().get())
+        .mcps(mcps, auditStamp, opContext.getRetrieverContext())
+        .retrieverContext(opContext.getRetrieverContext())
         .build();
   }
 

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java

@@ -168,13 +168,13 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
 
                   AspectsBatch aspectsBatch =
                       AspectsBatchImpl.builder()
-                          .retrieverContext(opContext.getRetrieverContext().get())
+                          .retrieverContext(opContext.getRetrieverContext())
                           .items(
                               batch
                                   .flatMap(
                                       ebeanAspectV2 ->
                                           EntityUtils.toSystemAspectFromEbeanAspects(
-                                              opContext.getRetrieverContext().get(),
+                                              opContext.getRetrieverContext(),
                                               Set.of(ebeanAspectV2))
                                               .stream())
                                   .map(
@@ -189,11 +189,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
                                               .auditStamp(systemAspect.getAuditStamp())
                                               .systemMetadata(
                                                   withAppSource(systemAspect.getSystemMetadata()))
-                                              .build(
-                                                  opContext
-                                                      .getRetrieverContext()
-                                                      .get()
-                                                      .getAspectRetriever()))
+                                              .build(opContext.getAspectRetriever()))
                                   .collect(Collectors.toList()))
                           .build();
 

datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/schemafield/GenerateSchemaFieldsFromSchemaMetadataStepTest.java

@@ -22,7 +22,6 @@
 import com.linkedin.upgrade.DataHubUpgradeState;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.RetrieverContext;
-import java.util.Optional;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -48,7 +47,7 @@ public void setup() {
     step =
         new GenerateSchemaFieldsFromSchemaMetadataStep(
             mockOpContext, mockEntityService, mockAspectDao, 10, 100, 1000);
-    when(mockOpContext.getRetrieverContext()).thenReturn(Optional.of(mockRetrieverContext));
+    when(mockOpContext.getRetrieverContext()).thenReturn(mockRetrieverContext);
   }
 
   /** Test to verify the correct step ID is returned. */

datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx

@@ -35,11 +35,9 @@ export class SchemaFieldPropertiesEntity implements Entity<SchemaFieldEntity> {
     // Currently unused.
     getPathName = () => 'schemaField';
 
-    // Currently unused.
-    getEntityName = () => 'schemaField';
+    getEntityName = () => 'Column';
 
-    // Currently unused.
-    getCollectionName = () => 'schemaFields';
+    getCollectionName = () => 'Columns';
 
     // Currently unused.
     renderProfile = (_: string) => <></>;

datahub-web-react/src/app/entity/shared/components/styled/DeprecationPill.tsx

@@ -19,8 +19,6 @@ const DeprecatedContainer = styled.div`
     justify-content: center;
     align-items: center;
     color: #cd0d24;
-    margin-left: 0px;
-    margin-right: 8px;
     padding-top: 8px;
     padding-bottom: 8px;
     padding-right: 4px;

datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleStringInput.tsx → datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/MultipleOpenEndedInput.tsx

@@ -4,6 +4,8 @@ import React from 'react';
 import styled from 'styled-components';
 import { ANTD_GRAY_V2 } from '../../../constants';
 
+const MultiStringWrapper = styled.div``;
+
 const StyledInput = styled(Input)`
     width: 75%;
     min-width: 350px;
@@ -29,10 +31,11 @@ const DeleteButton = styled(Button)`
 
 interface Props {
     selectedValues: any[];
+    inputType?: string;
     updateSelectedValues: (values: any[]) => void;
 }
 
-export default function MultipleStringInput({ selectedValues, updateSelectedValues }: Props) {
+export default function MultipleOpenEndedInput({ selectedValues, updateSelectedValues, inputType = 'text' }: Props) {
     function updateInput(text: string, index: number) {
         const updatedValues =
             selectedValues.length > 0 ? selectedValues.map((value, i) => (i === index ? text : value)) : [text];
@@ -53,14 +56,14 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
     }
 
     return (
-        <div>
+        <MultiStringWrapper>
             {selectedValues.length > 1 &&
                 selectedValues.map((selectedValue, index) => {
                     const key = `${index}`;
                     return (
                         <InputWrapper key={key}>
                             <StyledInput
-                                type="text"
+                                type={inputType}
                                 value={selectedValue}
                                 onChange={(e) => updateInput(e.target.value, index)}
                             />
@@ -70,14 +73,14 @@ export default function MultipleStringInput({ selectedValues, updateSelectedValu
                 })}
             {selectedValues.length <= 1 && (
                 <StyledInput
-                    type="text"
+                    type={inputType}
                     value={selectedValues[0] || ''}
                     onChange={(e) => updateInput(e.target.value, 0)}
                 />
             )}
             <StyledButton type="link" onClick={addNewValue}>
                 + Add More
             </StyledButton>
-        </div>
+        </MultiStringWrapper>
     );
 }

datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/NumberInput.tsx

@@ -1,7 +1,9 @@
 import { Input } from 'antd';
 import React, { ChangeEvent } from 'react';
 import styled from 'styled-components';
+import { PropertyCardinality } from '@src/types.generated';
 import { ANTD_GRAY_V2 } from '../../../constants';
+import MultipleOpenEndedInput from './MultipleOpenEndedInput';
 
 const StyledInput = styled(Input)`
     border: 1px solid ${ANTD_GRAY_V2[6]};
@@ -10,15 +12,31 @@ const StyledInput = styled(Input)`
 
 interface Props {
     selectedValues: any[];
+    cardinality?: PropertyCardinality | null;
     updateSelectedValues: (values: string[] | number[]) => void;
 }
 
-export default function NumberInput({ selectedValues, updateSelectedValues }: Props) {
+export default function NumberInput({ selectedValues, cardinality, updateSelectedValues }: Props) {
     function updateInput(event: ChangeEvent<HTMLInputElement>) {
         const number = Number(event.target.value);
         updateSelectedValues([number]);
     }
 
+    function updateMultipleValues(values: string[] | number[]) {
+        const numbers = values.map((v) => Number(v));
+        updateSelectedValues(numbers);
+    }
+
+    if (cardinality === PropertyCardinality.Multiple) {
+        return (
+            <MultipleOpenEndedInput
+                selectedValues={selectedValues}
+                updateSelectedValues={updateMultipleValues}
+                inputType="number"
+            />
+        );
+    }
+
     return (
         <StyledInput
             type="number"

datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StringInput.tsx

@@ -3,7 +3,7 @@ import React, { ChangeEvent } from 'react';
 import styled from 'styled-components';
 import { ANTD_GRAY_V2 } from '../../../constants';
 import { PropertyCardinality } from '../../../../../../types.generated';
-import MultipleStringInput from './MultipleStringInput';
+import MultipleOpenEndedInput from './MultipleOpenEndedInput';
 
 const StyledInput = styled(Input)`
     width: 75%;
@@ -24,8 +24,15 @@ export default function StringInput({ selectedValues, cardinality, updateSelecte
     }
 
     if (cardinality === PropertyCardinality.Multiple) {
-        return <MultipleStringInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
+        return <MultipleOpenEndedInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />;
     }
 
-    return <StyledInput type="text" value={selectedValues[0] || ''} onChange={updateInput} />;
+    return (
+        <StyledInput
+            type="text"
+            value={selectedValues[0] || ''}
+            onChange={updateInput}
+            data-testid="structured-property-string-value-input"
+        />
+    );
 }

datahub-web-react/src/app/entity/shared/components/styled/StructuredProperty/StructuredPropertyInput.tsx

@@ -60,7 +60,11 @@ export default function StructuredPropertyInput({
                 <DateInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
             )}
             {!allowedValues && valueType.info.type === StdDataType.Number && (
-                <NumberInput selectedValues={selectedValues} updateSelectedValues={updateSelectedValues} />
+                <NumberInput
+                    selectedValues={selectedValues}
+                    cardinality={cardinality}
+                    updateSelectedValues={updateSelectedValues}
+                />
             )}
             {!allowedValues && valueType.info.type === StdDataType.Urn && (
                 <UrnInput

datahub-web-react/src/app/entity/shared/containers/profile/header/EntityHeader.tsx

@@ -24,6 +24,7 @@ const TitleWrapper = styled.div`
     display: flex;
     justify-content: left;
     align-items: center;
+    gap: 8px;
 
     .ant-typography-edit-content {
         padding-top: 7px;

datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx

@@ -8,8 +8,6 @@ import { useEntityData, useRefetch } from '../../../EntityContext';
 import { useGlossaryEntityData } from '../../../GlossaryEntityContext';
 
 export const EntityTitle = styled(Typography.Title)`
-    margin-right: 10px;
-
     &&& {
         margin-bottom: 0;
         word-break: break-all;

datahub-web-react/src/app/entity/shared/tabs/Properties/__tests__/useStructuredProperties.test.ts

@@ -0,0 +1,87 @@
+import { identifyAndAddParentRows } from '../useStructuredProperties';
+
+describe('identifyAndAddParentRows', () => {
+    it('should not return parent rows when there are none', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'test1' },
+            { displayName: 'test2', qualifiedName: 'test2' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([]);
+    });
+
+    it('should not return parent rows when another row starts with the same letters but is a different token', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one' },
+            { displayName: 'test2', qualifiedName: 'testingAgain.two' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([]);
+    });
+
+    it('should return parent rows properly', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one' },
+            { displayName: 'test2', qualifiedName: 'testing.two' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', childrenCount: 3 },
+        ]);
+    });
+
+    it('should return parent rows properly with multiple layers of nesting', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.1' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.2' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.b' },
+            { displayName: 'test1', qualifiedName: 'testing.one.three' },
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 6 },
+            { displayName: 'testing.one', qualifiedName: 'testing.one', isParentRow: true, childrenCount: 4 },
+            { displayName: 'testing.one.two', qualifiedName: 'testing.one.two', isParentRow: true, childrenCount: 3 },
+            {
+                displayName: 'testing.one.two.a',
+                qualifiedName: 'testing.one.two.a',
+                isParentRow: true,
+                childrenCount: 2,
+            },
+        ]);
+    });
+
+    it('should return parent rows properly with multiple layers of nesting regardless of order', () => {
+        const propertyRows = [
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.1' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+            { displayName: 'test1', qualifiedName: 'testing.one.three' },
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.b' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test1', qualifiedName: 'testing.one.two.a.2' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 6 },
+            { displayName: 'testing.one', qualifiedName: 'testing.one', isParentRow: true, childrenCount: 4 },
+            { displayName: 'testing.one.two', qualifiedName: 'testing.one.two', isParentRow: true, childrenCount: 3 },
+            {
+                displayName: 'testing.one.two.a',
+                qualifiedName: 'testing.one.two.a',
+                isParentRow: true,
+                childrenCount: 2,
+            },
+        ]);
+    });
+
+    it('should return parent rows properly with simpler layers of nesting', () => {
+        const propertyRows = [
+            { displayName: 'test2', qualifiedName: 'testing.two.c.d' },
+            { displayName: 'test3', qualifiedName: 'testing.three' },
+            { displayName: 'test3', qualifiedName: 'testParent' },
+        ];
+        expect(identifyAndAddParentRows(propertyRows)).toMatchObject([
+            { displayName: 'testing', qualifiedName: 'testing', isParentRow: true, childrenCount: 2 },
+        ]);
+    });
+});

datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx

@@ -122,10 +122,10 @@ export function identifyAndAddParentRows(rows?: Array<PropertyRow>): Array<Prope
         // that would tell us to nest. If the count is not equal, we should nest the child properties.
         for (let index = 0; index < substrings.length; index++) {
             const token = substrings[index];
-            const currentCount = qualifiedNames.filter((name) => name.startsWith(token)).length;
+            const currentCount = qualifiedNames.filter((name) => name.startsWith(`${token}.`)).length;
 
-            // If we're at the beginning of the path and there is no nesting, break
-            if (index === 0 && currentCount === 1) {
+            // If there's only one child, don't nest it
+            if (currentCount === 1) {
                 break;
             }