Merge branch 'master' into master

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/assertion/AssertionMapper.java

@@ -66,6 +66,7 @@ private static com.linkedin.datahub.graphql.generated.AssertionInfo mapAssertion
           mapDatasetAssertionInfo(gmsAssertionInfo.getDatasetAssertion());
       assertionInfo.setDatasetAssertion(datasetAssertion);
     }
+    assertionInfo.setDescription(gmsAssertionInfo.getDescription());
     return assertionInfo;
   }
 

datahub-graphql-core/src/main/resources/entity.graphql

@@ -6803,6 +6803,11 @@ type AssertionInfo {
     Dataset-specific assertion information
     """
     datasetAssertion: DatasetAssertionInfo
+
+    """
+    An optional human-readable description of the assertion
+    """
+    description: String
 }
 
 """

datahub-web-react/src/App.tsx

@@ -1,20 +1,19 @@
-import React, { useEffect, useState } from 'react';
+import React from 'react';
 import Cookies from 'js-cookie';
 import { message } from 'antd';
 import { BrowserRouter as Router } from 'react-router-dom';
 import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client';
 import { onError } from '@apollo/client/link/error';
-import { ThemeProvider } from 'styled-components';
 import { Helmet, HelmetProvider } from 'react-helmet-async';
 import './App.less';
 import { Routes } from './app/Routes';
-import { Theme } from './conf/theme/types';
-import defaultThemeConfig from './conf/theme/theme_light.config.json';
 import { PageRoutes } from './conf/Global';
 import { isLoggedInVar } from './app/auth/checkAuthStatus';
 import { GlobalCfg } from './conf';
 import possibleTypesResult from './possibleTypes.generated';
 import { ErrorCodes } from './app/shared/constants';
+import CustomThemeProvider from './CustomThemeProvider';
+import { useCustomTheme } from './customThemeContext';
 
 /*
     Construct Apollo Client
@@ -71,33 +70,16 @@ const client = new ApolloClient({
 });
 
 export const InnerApp: React.VFC = () => {
-    const [dynamicThemeConfig, setDynamicThemeConfig] = useState<Theme>(defaultThemeConfig);
-
-    useEffect(() => {
-        if (import.meta.env.DEV) {
-            import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
-                setDynamicThemeConfig(theme);
-            });
-        } else {
-            // Send a request to the server to get the theme config.
-            fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
-                .then((response) => response.json())
-                .then((theme) => {
-                    setDynamicThemeConfig(theme);
-                });
-        }
-    }, []);
-
     return (
         <HelmetProvider>
-            <Helmet>
-                <title>{dynamicThemeConfig.content.title}</title>
-            </Helmet>
-            <ThemeProvider theme={dynamicThemeConfig}>
+            <CustomThemeProvider>
+                <Helmet>
+                    <title>{useCustomTheme().theme?.content.title}</title>
+                </Helmet>
                 <Router>
                     <Routes />
                 </Router>
-            </ThemeProvider>
+            </CustomThemeProvider>
         </HelmetProvider>
     );
 };

datahub-web-react/src/CustomThemeProvider.tsx

@@ -0,0 +1,32 @@
+import React, { useEffect, useState } from 'react';
+import { ThemeProvider } from 'styled-components';
+import { Theme } from './conf/theme/types';
+import defaultThemeConfig from './conf/theme/theme_light.config.json';
+import { CustomThemeContext } from './customThemeContext';
+
+const CustomThemeProvider = ({ children }: { children: React.ReactNode }) => {
+    const [currentTheme, setTheme] = useState<Theme>(defaultThemeConfig);
+
+    useEffect(() => {
+        if (import.meta.env.DEV) {
+            import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
+                setTheme(theme);
+            });
+        } else {
+            // Send a request to the server to get the theme config.
+            fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
+                .then((response) => response.json())
+                .then((theme) => {
+                    setTheme(theme);
+                });
+        }
+    }, []);
+
+    return (
+        <CustomThemeContext.Provider value={{ theme: currentTheme, updateTheme: setTheme }}>
+            <ThemeProvider theme={currentTheme}>{children}</ThemeProvider>
+        </CustomThemeContext.Provider>
+    );
+};
+
+export default CustomThemeProvider;

datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/Assertions.tsx

@@ -35,6 +35,8 @@ const getAssertionsStatusSummary = (assertions: Array<Assertion>) => {
 
 /**
  * Component used for rendering the Validations Tab on the Dataset Page.
+ *
+ * TODO: Note that only the legacy DATASET assertions are supported for viewing as of today.
  */
 export const Assertions = () => {
     const { urn, entityData } = useEntityData();
@@ -47,7 +49,9 @@ export const Assertions = () => {
     const assertions =
         (combinedData && combinedData.dataset?.assertions?.assertions?.map((assertion) => assertion as Assertion)) ||
         [];
-    const filteredAssertions = assertions.filter((assertion) => !removedUrns.includes(assertion.urn));
+    const filteredAssertions = assertions.filter(
+        (assertion) => !removedUrns.includes(assertion.urn) && !!assertion.info?.datasetAssertion,
+    );
 
     // Pre-sort the list of assertions based on which has been most recently executed.
     assertions.sort(sortAssertions);

datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionDescription.tsx

@@ -19,6 +19,7 @@ const ViewLogicButton = styled(Button)`
 `;
 
 type Props = {
+    description?: string;
     assertionInfo: DatasetAssertionInfo;
 };
 
@@ -319,18 +320,20 @@ const TOOLTIP_MAX_WIDTH = 440;
  *
  * For example, Column 'X' values are in [1, 2, 3]
  */
-export const DatasetAssertionDescription = ({ assertionInfo }: Props) => {
+export const DatasetAssertionDescription = ({ description, assertionInfo }: Props) => {
     const { scope, aggregation, fields, operator, parameters, nativeType, nativeParameters, logic } = assertionInfo;
     const [isLogicVisible, setIsLogicVisible] = useState(false);
     /**
      * Build a description component from a) input (aggregation, inputs) b) the operator text
      */
-    const description = (
+    const descriptionFragment = (
         <>
-            <Typography.Text>
-                {getAggregationText(scope, aggregation, fields)}{' '}
-                {getOperatorText(operator, parameters || undefined, nativeType || undefined)}
-            </Typography.Text>
+            {description || (
+                <Typography.Text>
+                    {getAggregationText(scope, aggregation, fields)}{' '}
+                    {getOperatorText(operator, parameters || undefined, nativeType || undefined)}
+                </Typography.Text>
+            )}
         </>
     );
 
@@ -349,7 +352,7 @@ export const DatasetAssertionDescription = ({ assertionInfo }: Props) => {
                 </>
             }
         >
-            <div>{description}</div>
+            <div>{descriptionFragment}</div>
             {logic && (
                 <div>
                     <ViewLogicButton onClick={() => setIsLogicVisible(true)} type="link">

datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx

@@ -83,6 +83,7 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => {
         type: assertion.info?.type,
         platform: assertion.platform,
         datasetAssertionInfo: assertion.info?.datasetAssertion,
+        description: assertion.info?.description,
         lastExecTime: assertion.runEvents?.runEvents?.length && assertion.runEvents.runEvents[0].timestampMillis,
         lastExecResult:
             assertion.runEvents?.runEvents?.length &&
@@ -101,6 +102,7 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => {
                 const resultColor = (record.lastExecResult && getResultColor(record.lastExecResult)) || 'default';
                 const resultText = (record.lastExecResult && getResultText(record.lastExecResult)) || 'No Evaluations';
                 const resultIcon = (record.lastExecResult && getResultIcon(record.lastExecResult)) || <StopOutlined />;
+                const { description } = record;
                 return (
                     <ResultContainer>
                         <div>
@@ -111,7 +113,10 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => {
                                 </Tag>
                             </Tooltip>
                         </div>
-                        <DatasetAssertionDescription assertionInfo={record.datasetAssertionInfo} />
+                        <DatasetAssertionDescription
+                            description={description}
+                            assertionInfo={record.datasetAssertionInfo}
+                        />
                     </ResultContainer>
                 );
             },
@@ -146,12 +151,7 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => {
                     <Button onClick={() => onDeleteAssertion(record.urn)} type="text" shape="circle" danger>
                         <DeleteOutlined />
                     </Button>
-                    <Dropdown
-                        overlay={
-                            <AssertionMenu urn={record.urn}/>
-                        }
-                        trigger={['click']}
-                    >
+                    <Dropdown overlay={<AssertionMenu urn={record.urn} />} trigger={['click']}>
                         <StyledMoreOutlined />
                     </Dropdown>
                 </ActionButtonContainer>

datahub-web-react/src/customThemeContext.tsx

@@ -0,0 +1,10 @@
+import React, { useContext } from 'react';
+
+export const CustomThemeContext = React.createContext<{
+    theme: any;
+    updateTheme: (theme: any) => void;
+}>({ theme: undefined, updateTheme: (_) => null });
+
+export function useCustomTheme() {
+    return useContext(CustomThemeContext);
+}

datahub-web-react/src/graphql/assertion.graphql

@@ -46,6 +46,7 @@ fragment assertionDetails on Assertion {
             }
             logic
         }
+        description
     }
 }
 

metadata-ingestion/developing.md

@@ -10,7 +10,7 @@ Also take a look at the guide to [adding a source](./adding-source.md).
 ### Requirements
 
 1. Python 3.7+ must be installed in your host environment.
-2. Java8 (gradle won't work with newer versions)
+2. Java 17 (gradle won't work with newer or older versions)
 4. On Debian/Ubuntu: `sudo apt install python3-dev python3-venv`
 5. On Fedora (if using LDAP source integration): `sudo yum install openldap-devel`
 

metadata-ingestion/docs/sources/metabase/metabase.md

@@ -19,4 +19,4 @@ The key in this map must be string, not integer although  Metabase API provides
 If `database_id_to_instance_map` is not specified, `platform_instance_map` is used for platform instance mapping. If none of the above are specified, platform instance is not used when constructing `urn` when searching for dataset relations.
 ## Compatibility
 
-Metabase version [v0.41.2](https://www.metabase.com/start/oss/)
+Metabase version [v0.48.3](https://www.metabase.com/start/oss/)

metadata-ingestion/src/datahub/ingestion/source/metabase.py

@@ -90,10 +90,17 @@ class MetabaseSource(Source):
     """
     This plugin extracts Charts, dashboards, and associated metadata. This plugin is in beta and has only been tested
     on PostgreSQL and H2 database.
-    ### Dashboard
 
-    [/api/dashboard](https://www.metabase.com/docs/latest/api-documentation.html#dashboard) endpoint is used to
-    retrieve the following dashboard information.
+    ### Collection
+
+    [/api/collection](https://www.metabase.com/docs/latest/api/collection) endpoint is used to
+    retrieve the available collections.
+
+    [/api/collection/<COLLECTION_ID>/items?models=dashboard](https://www.metabase.com/docs/latest/api/collection#get-apicollectioniditems) endpoint is used to retrieve a given collection and list their dashboards.
+
+     ### Dashboard
+
+    [/api/dashboard/<DASHBOARD_ID>](https://www.metabase.com/docs/latest/api/dashboard) endpoint is used to retrieve a given Dashboard and grab its information.
 
     - Title and description
     - Last edited by
@@ -187,19 +194,29 @@ def close(self) -> None:
 
     def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]:
         try:
-            dashboard_response = self.session.get(
-                f"{self.config.connect_uri}/api/dashboard"
+            collections_response = self.session.get(
+                f"{self.config.connect_uri}/api/collection/"
             )
-            dashboard_response.raise_for_status()
-            dashboards = dashboard_response.json()
+            collections_response.raise_for_status()
+            collections = collections_response.json()
 
-            for dashboard_info in dashboards:
-                dashboard_snapshot = self.construct_dashboard_from_api_data(
-                    dashboard_info
+            for collection in collections:
+                collection_dashboards_response = self.session.get(
+                    f"{self.config.connect_uri}/api/collection/{collection['id']}/items?models=dashboard"
                 )
-                if dashboard_snapshot is not None:
-                    mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
-                    yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
+                collection_dashboards_response.raise_for_status()
+                collection_dashboards = collection_dashboards_response.json()
+
+                if not collection_dashboards.get("data"):
+                    continue
+
+                for dashboard_info in collection_dashboards.get("data"):
+                    dashboard_snapshot = self.construct_dashboard_from_api_data(
+                        dashboard_info
+                    )
+                    if dashboard_snapshot is not None:
+                        mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
+                        yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
 
         except HTTPError as http_error:
             self.report.report_failure(
@@ -254,10 +271,10 @@ def construct_dashboard_from_api_data(
         )
 
         chart_urns = []
-        cards_data = dashboard_details.get("ordered_cards", "{}")
+        cards_data = dashboard_details.get("dashcards", {})
         for card_info in cards_data:
             chart_urn = builder.make_chart_urn(
-                self.platform, card_info.get("card_id", "")
+                self.platform, card_info.get("card").get("id", "")
             )
             chart_urns.append(chart_urn)