From 5c10d2f262980063ac836ce403b7fbf5391a703e Mon Sep 17 00:00:00 2001 From: Junda Yang Date: Thu, 25 Apr 2019 15:20:47 -0700 Subject: [PATCH] add is_view field to TableDetailAPI payload (#29) * add is_view to payload * add test * convert str(is_view) to boolean * fix tests * nit * bump up amundsenmetadatalibrary version --- metadata/metadata_service/api/table.py | 3 +- .../metadata_service/entity/table_detail.py | 19 ++++---- .../metadata_service/proxy/neo4j_proxy.py | 5 +- metadata/setup.py | 2 +- metadata/tests/unit/proxy/test_neo4j_proxy.py | 47 ++++++++++++++++++- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/metadata/metadata_service/api/table.py b/metadata/metadata_service/api/table.py index 6bcaf9a096..3b6f48759e 100644 --- a/metadata/metadata_service/api/table.py +++ b/metadata/metadata_service/api/table.py @@ -76,7 +76,8 @@ 'watermarks': fields.List(fields.Nested(watermark_fields)), 'table_writer': fields.Nested(table_writer_fields), # Optional 'last_updated_timestamp': fields.Integer, # Optional - 'source': fields.Nested(source_fields) # Optional + 'source': fields.Nested(source_fields), # Optional + 'is_view': fields.Boolean # Optional } diff --git a/metadata/metadata_service/entity/table_detail.py b/metadata/metadata_service/entity/table_detail.py index 7cccb98906..df58409b6c 100644 --- a/metadata/metadata_service/entity/table_detail.py +++ b/metadata/metadata_service/entity/table_detail.py @@ -137,14 +137,15 @@ def __init__(self, *, schema: str, name: str, tags: Iterable[Tag] =(), - table_readers: Iterable[Reader] =(), - description: Optional[str] =None, + table_readers: Iterable[Reader] = (), + description: Optional[str] = None, columns: Iterable[Column], - owners: Iterable[User] =(), - watermarks: Iterable[Watermark] =(), - table_writer: Optional[Application] =None, + owners: Iterable[User] = (), + watermarks: Iterable[Watermark] = (), + table_writer: Optional[Application] = None, last_updated_timestamp: Optional[int], - source: Optional[Source] = None + source: Optional[Source] = None, + is_view: Optional[bool] = None, ) -> None: self.database = database @@ -160,13 +161,15 @@ def __init__(self, *, self.table_writer = table_writer self.last_updated_timestamp = last_updated_timestamp self.source = source + self.is_view = is_view or False def __repr__(self) -> str: return """Table(database={!r}, cluster={!r}, schema={!r}, name={!r}, tags={!r}, table_readers={!r}, description={!r}, columns={!r}, owners={!r}, watermarks={!r}, table_writer={!r}, - last_updated_timestamp={!r}, source={!r})"""\ + last_updated_timestamp={!r}, source={!r}, is_view={!r})"""\ .format(self.database, self.cluster, self.schema, self.name, self.tags, self.table_readers, self.description, self.columns, self.owners, self.watermarks, - self.table_writer, self.last_updated_timestamp, self.source) + self.table_writer, self.last_updated_timestamp, + self.source, self.is_view) diff --git a/metadata/metadata_service/proxy/neo4j_proxy.py b/metadata/metadata_service/proxy/neo4j_proxy.py index 65458df1f1..3566d5d9b4 100644 --- a/metadata/metadata_service/proxy/neo4j_proxy.py +++ b/metadata/metadata_service/proxy/neo4j_proxy.py @@ -81,7 +81,8 @@ def get_table(self, *, table_uri: str) -> Table: watermarks=wmk_results, table_writer=table_writer, last_updated_timestamp=timestamp_value, - source=source) + source=source, + is_view=self._safe_get(last_neo4j_record, 'tbl', 'is_view')) return table @@ -233,7 +234,7 @@ def _safe_get(self, dct, *keys): """ for key in keys: dct = dct.get(key) - if not dct: + if dct is None: return None return dct diff --git a/metadata/setup.py b/metadata/setup.py index d8db6b836b..5fdc4852e6 100644 --- a/metadata/setup.py +++ b/metadata/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -__version__ = '1.0.4' +__version__ = '1.0.5' setup( diff --git a/metadata/tests/unit/proxy/test_neo4j_proxy.py b/metadata/tests/unit/proxy/test_neo4j_proxy.py index bcf0dec765..92a4901422 100644 --- a/metadata/tests/unit/proxy/test_neo4j_proxy.py +++ b/metadata/tests/unit/proxy/test_neo4j_proxy.py @@ -146,7 +146,52 @@ def test_get_table(self) -> None: id=self.table_writer['id']), last_updated_timestamp=1, source=Source(source='/source_file_loc', - source_type='github')) + source_type='github'), + is_view=False) + + self.assertEqual(str(expected), str(table)) + + def test_get_table_view_only(self) -> None: + col_usage_return_value = copy.deepcopy(self.col_usage_return_value) + for col in col_usage_return_value: + col['tbl']['is_view'] = True + + with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jProxy, '_execute_cypher_query') as mock_execute: + mock_execute.side_effect = [col_usage_return_value, [], self.table_level_return_value] + + neo4j_proxy = Neo4jProxy(host='DOES_NOT_MATTER', port=0000) + table = neo4j_proxy.get_table(table_uri='dummy_uri') + + expected = Table(database='hive', cluster='gold', schema='foo_schema', name='foo_table', + tags=[Tag(tag_name='test', tag_type='default')], + table_readers=[], description='foo description', + watermarks=[Watermark(watermark_type='high_watermark', + partition_key='ds', + partition_value='fake_value', + create_time='fake_time'), + Watermark(watermark_type='low_watermark', + partition_key='ds', + partition_value='fake_value', + create_time='fake_time')], + columns=[Column(name='bar_id_1', description='bar col description', col_type='varchar', + sort_order=0, stats=[Statistics(start_epoch=1, + end_epoch=1, + stat_type='avg', + stat_val='1')]), + Column(name='bar_id_2', description='bar col2 description', col_type='bigint', + sort_order=1, stats=[Statistics(start_epoch=2, + end_epoch=2, + stat_type='avg', + stat_val='2')])], + owners=[User(email='tester@lyft.com')], + table_writer=Application(application_url=self.table_writer['application_url'], + description=self.table_writer['description'], + name=self.table_writer['name'], + id=self.table_writer['id']), + last_updated_timestamp=1, + source=Source(source='/source_file_loc', + source_type='github'), + is_view=True) self.assertEqual(str(expected), str(table))