Integrate caching for Iceberg table fetch

Netflix · Feb 7, 2022 · 5f7aa00 · 5f7aa00
1 parent 7fb2477
commit 5f7aa00
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 4 deletions.
diff --git a/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java b/...ris/src/main/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableService.java
@@ -26,6 +26,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
+import org.springframework.cache.annotation.Cacheable;
 import org.springframework.dao.DataIntegrityViolationException;
 
 import javax.annotation.Nullable;
@@ -140,9 +141,8 @@ public TableInfo get(final ConnectorRequestContext requestContext, final Qualifi
                 .orElseThrow(() -> new TableNotFoundException(name));
             final TableInfo info = polarisTableMapper.toInfo(polarisTableEntity);
             final String tableLoc = HiveTableUtil.getIcebergTableMetadataLocation(info);
-            final IcebergTableWrapper icebergTable = icebergTableHandler.getIcebergTable(
-                name, tableLoc, requestContext.isIncludeMetadata());
-            return connectorConverter.fromIcebergTableToTableInfo(name, icebergTable, tableLoc, info);
+            return getIcebergTable(name, tableLoc, info,
+                requestContext.isIncludeMetadata(), connectorContext.getConfig().isIcebergCacheEnabled());
         } catch (TableNotFoundException | IllegalArgumentException exception) {
             log.error(String.format("Not found exception for polaris table %s", name), exception);
             throw exception;
@@ -320,4 +320,24 @@ public List<TableInfo> list(
             throw new ConnectorException(msg, exception);
         }
     }
+
+    /**
+     * Return the table metadata from cache if exists else make the iceberg call and refresh it.
+     * @param tableName             table name
+     * @param tableMetadataLocation table metadata location
+     * @param info                  table info stored in hive metastore
+     * @param includeInfoDetails    if true, will include more details like the manifest file content
+     * @param useCache              true, if table can be retrieved from cache
+     * @return TableInfo
+     */
+    @Cacheable(key = "'iceberg.table.' + #includeInfoDetails + '.' + #tableMetadataLocation", condition = "#useCache")
+    public TableInfo getIcebergTable(final QualifiedName tableName,
+                                     final String tableMetadataLocation,
+                                     final TableInfo info,
+                                     final boolean includeInfoDetails,
+                                     final boolean useCache) {
+        final IcebergTableWrapper icebergTable =
+            this.icebergTableHandler.getIcebergTable(tableName, tableMetadataLocation, includeInfoDetails);
+        return connectorConverter.fromIcebergTableToTableInfo(tableName, icebergTable, tableMetadataLocation, info);
+    }
 }
diff --git a/...src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java b/...src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java
@@ -339,4 +339,3 @@ public void testUpdateTableAccept() {
         Assert.assertEquals(tableResult1.getMetadata().get("metadata_location"), location1);
     }
 }
-
Original file line number	Diff line number	Diff line change
Expand Up		@@ -339,4 +339,3 @@ public void testUpdateTableAccept() {
		Assert.assertEquals(tableResult1.getMetadata().get("metadata_location"), location1);
		}
		}