From 232317733f59f6647768b50ee337e61a4375d1b5 Mon Sep 17 00:00:00 2001 From: pthirun Date: Mon, 16 Dec 2024 17:10:02 -0800 Subject: [PATCH] add integration test to cover store changes --- .../venice/meta/ReadOnlyStoreTest.java | 14 +- .../TestServerStorePropertiesEndpoint.java | 147 ++++++++++++++---- 2 files changed, 122 insertions(+), 39 deletions(-) diff --git a/internal/venice-common/src/test/java/com/linkedin/venice/meta/ReadOnlyStoreTest.java b/internal/venice-common/src/test/java/com/linkedin/venice/meta/ReadOnlyStoreTest.java index 02467f5668..8ca379a50e 100644 --- a/internal/venice-common/src/test/java/com/linkedin/venice/meta/ReadOnlyStoreTest.java +++ b/internal/venice-common/src/test/java/com/linkedin/venice/meta/ReadOnlyStoreTest.java @@ -21,10 +21,10 @@ public class ReadOnlyStoreTest { private static final Logger LOGGER = LogManager.getLogger(ReadOnlyStoreTest.class); - private static Random RANDOM; + private Random RANDOM; @BeforeClass - public static void setup() { + public void setupReadOnlyStore() { long seed = System.nanoTime(); RANDOM = new Random(seed); LOGGER.info("Random seed set: {}", seed); @@ -106,7 +106,7 @@ public void testCloneStoreProperties() { assertEquals(storeProperties.getNearlineProducerCountPerWriter(), store.getNearlineProducerCountPerWriter()); } - private static ZKStore populateZKStore(ZKStore store) { + private ZKStore populateZKStore(ZKStore store) { store.setCurrentVersion(RANDOM.nextInt()); store.setPartitionCount(RANDOM.nextInt()); store.setLowWatermark(RANDOM.nextLong()); @@ -158,7 +158,7 @@ private static ZKStore populateZKStore(ZKStore store) { return store; } - private static void assertEqualHybridConfig(StoreHybridConfig actual, HybridStoreConfig expected) { + private void assertEqualHybridConfig(StoreHybridConfig actual, HybridStoreConfig expected) { assertEquals(actual.getRewindTimeInSeconds(), expected.getRewindTimeInSeconds()); assertEquals(actual.getOffsetLagThresholdToGoOnline(), expected.getOffsetLagThresholdToGoOnline()); assertEquals( @@ -188,13 +188,13 @@ private static void assertEqualViewConfig( } } - private static void assertEqualsETLStoreConfig(StoreETLConfig actual, ETLStoreConfig expected) { + private void assertEqualsETLStoreConfig(StoreETLConfig actual, ETLStoreConfig expected) { assertEquals(actual.getEtledUserProxyAccount(), expected.getEtledUserProxyAccount()); assertEquals(actual.getFutureVersionETLEnabled(), expected.isFutureVersionETLEnabled()); assertEquals(actual.getRegularVersionETLEnabled(), expected.isFutureVersionETLEnabled()); } - private static void assertEqualsPartitionerConfig(StorePartitionerConfig actual, PartitionerConfig expected) { + private void assertEqualsPartitionerConfig(StorePartitionerConfig actual, PartitionerConfig expected) { assertEquals(actual.getPartitionerClass(), expected.getPartitionerClass()); assertEquals(actual.getAmplificationFactor(), expected.getAmplificationFactor()); @@ -207,7 +207,7 @@ private static void assertEqualsPartitionerConfig(StorePartitionerConfig actual, } } - private static void assertEqualsSystemStores( + private void assertEqualsSystemStores( Map actual, Map expected) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestServerStorePropertiesEndpoint.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestServerStorePropertiesEndpoint.java index 00ebbda19b..96ed090073 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestServerStorePropertiesEndpoint.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestServerStorePropertiesEndpoint.java @@ -2,82 +2,165 @@ import static org.testng.Assert.*; -import com.fasterxml.jackson.databind.ObjectWriter; import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.client.store.transport.TransportClient; import com.linkedin.venice.client.store.transport.TransportClientResponse; +import com.linkedin.venice.controller.Admin; import com.linkedin.venice.fastclient.utils.AbstractClientEndToEndSetup; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.QueryAction; import com.linkedin.venice.meta.Store; import com.linkedin.venice.metadata.response.StorePropertiesResponseRecord; +import com.linkedin.venice.schema.SchemaEntry; import com.linkedin.venice.security.SSLFactory; import com.linkedin.venice.serializer.FastSerializerDeserializerFactory; import com.linkedin.venice.serializer.RecordDeserializer; -import com.linkedin.venice.utils.ObjectMapperFactory; import com.linkedin.venice.utils.SslUtils; +import java.util.Map; import java.util.Optional; +import java.util.Random; import org.apache.avro.Schema; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; public class TestServerStorePropertiesEndpoint extends AbstractClientEndToEndSetup { + private static final Logger LOGGER = LogManager.getLogger(TestServerStorePropertiesEndpoint.class); + + private Random RANDOM; + + private Optional sslFactory; + private VeniceServerWrapper veniceServerWrapper; + private String serverUrl; + + @BeforeClass + public void beforeClassServerStoreProperties() { + long seed = System.nanoTime(); + RANDOM = new Random(seed); + LOGGER.info("Random seed set: {}", seed); + } + + @BeforeMethod + public void beforeMethodServerStoreProperties() { + sslFactory = Optional.of(SslUtils.getVeniceLocalSslFactory()); + veniceServerWrapper = veniceCluster.getVeniceServers().stream().findAny().get(); + serverUrl = "https://" + veniceServerWrapper.getHost() + ":" + veniceServerWrapper.getPort(); + } + + @AfterMethod + public void afterMethodServerStoreProperties() { + veniceServerWrapper.close(); + veniceCluster.close(); + } + @Test(timeOut = TIME_OUT) public void testRequestBasedStoreProperties() throws Exception { + Admin admin = veniceCluster.getLeaderVeniceController().getVeniceAdmin(); + StorePropertiesResponseRecord record = getStorePropertiesResponseRecord(storeName); + assertStorePropertiesResponseRecord(record, admin, storeName); + } + + @Test(timeOut = TIME_OUT) + public void testRequestBasedStorePropertiesWithStoreChanges() throws Exception { + + String clusterName = veniceCluster.getClusterName(); + String owner = Long.toString(RANDOM.nextLong()); + int largestUsedVersion = RANDOM.nextInt(); + + veniceCluster.getLeaderVeniceController().getVeniceAdmin().setStoreOwner(clusterName, storeName, owner); + veniceCluster.getLeaderVeniceController() + .getVeniceAdmin() + .setStoreLargestUsedVersion(clusterName, storeName, largestUsedVersion); - // SSL - Optional sslFactory = Optional.of(SslUtils.getVeniceLocalSslFactory()); + StorePropertiesResponseRecord record = getStorePropertiesResponseRecord(storeName); - // Server - VeniceServerWrapper veniceServerWrapper = veniceCluster.getVeniceServers().stream().findAny().get(); - String serverUrl = "https://" + veniceServerWrapper.getHost() + ":" + veniceServerWrapper.getPort(); + assertEquals(record.storeMetaValue.storeProperties.owner.toString(), owner); + assertEquals(record.storeMetaValue.storeProperties.largestUsedVersionNumber, largestUsedVersion); + + Admin admin = veniceCluster.getLeaderVeniceController().getVeniceAdmin(); + assertStorePropertiesResponseRecord(record, admin, storeName); + } + + private StorePropertiesResponseRecord getStorePropertiesResponseRecord(String _storeName) throws Exception { + StorePropertiesResponseRecord record; // Request - ClientConfig clientConfig = ClientConfig.defaultGenericClientConfig(storeName).setVeniceURL(serverUrl); + ClientConfig clientConfig = ClientConfig.defaultGenericClientConfig(_storeName).setVeniceURL(serverUrl); clientConfig.setSslFactory(sslFactory.get()); TransportClient transportClient = ClientFactory.getTransportClient(clientConfig); - String requestUrl = QueryAction.STORE_PROPERTIES.toString().toLowerCase() + "/" + storeName; + String requestUrl = QueryAction.STORE_PROPERTIES.toString().toLowerCase() + "/" + _storeName; TransportClientResponse response = transportClient.get(requestUrl).get(); // Deserialize Schema writerSchema = StorePropertiesResponseRecord.SCHEMA$; RecordDeserializer recordDeserializer = FastSerializerDeserializerFactory .getFastAvroSpecificDeserializer(writerSchema, StorePropertiesResponseRecord.class); - StorePropertiesResponseRecord record = recordDeserializer.deserialize(response.getBody()); + record = recordDeserializer.deserialize(response.getBody()); + + return record; + } - ObjectWriter jsonWriter = ObjectMapperFactory.getInstance().writerWithDefaultPrettyPrinter(); - Object printObject = ObjectMapperFactory.getInstance().readValue(record.toString(), Object.class); - System.out.println(jsonWriter.writeValueAsString(printObject)); + private void assertStorePropertiesResponseRecord( + StorePropertiesResponseRecord record, + Admin admin, + String _storeName) { - // Assert - Store expectedStore = - veniceCluster.getLeaderVeniceController().getVeniceAdmin().getStore(veniceCluster.getClusterName(), storeName); + String clusterName = veniceCluster.getClusterName(); + Store store = admin.getStore(clusterName, _storeName); + // Assert assertNotNull(record); assertNotNull(record.storeMetaValue); assertNotNull(record.storeMetaValue.storeProperties); - assertEquals(record.storeMetaValue.storeProperties.name.toString(), expectedStore.getName()); - assertEquals(record.storeMetaValue.storeProperties.owner.toString(), expectedStore.getOwner()); - assertEquals(record.storeMetaValue.storeProperties.createdTime, expectedStore.getCreatedTime()); - assertEquals(record.storeMetaValue.storeProperties.currentVersion, expectedStore.getCurrentVersion()); - assertEquals(record.storeMetaValue.storeProperties.storageQuotaInByte, expectedStore.getStorageQuotaInByte()); - assertEquals(record.storeMetaValue.storeProperties.readQuotaInCU, expectedStore.getReadQuotaInCU()); - assertEquals( - record.storeMetaValue.storeProperties.largestUsedVersionNumber, - expectedStore.getLargestUsedVersionNumber()); + + // Store Properties + assertEquals(record.storeMetaValue.storeProperties.name.toString(), store.getName()); + assertEquals(record.storeMetaValue.storeProperties.owner.toString(), store.getOwner()); + assertEquals(record.storeMetaValue.storeProperties.createdTime, store.getCreatedTime()); + assertEquals(record.storeMetaValue.storeProperties.currentVersion, store.getCurrentVersion()); + assertEquals(record.storeMetaValue.storeProperties.partitionCount, store.getPartitionCount()); + assertEquals(record.storeMetaValue.storeProperties.lowWatermark, store.getLowWatermark()); + assertEquals(record.storeMetaValue.storeProperties.enableWrites, store.isEnableWrites()); + assertEquals(record.storeMetaValue.storeProperties.enableReads, store.isEnableReads()); + assertEquals(record.storeMetaValue.storeProperties.storageQuotaInByte, store.getStorageQuotaInByte()); + assertEquals(record.storeMetaValue.storeProperties.readQuotaInCU, store.getReadQuotaInCU()); + assertEquals(record.storeMetaValue.storeProperties.batchGetLimit, store.getBatchGetLimit()); + assertEquals(record.storeMetaValue.storeProperties.largestUsedVersionNumber, store.getLargestUsedVersionNumber()); assertEquals( record.storeMetaValue.storeProperties.latestVersionPromoteToCurrentTimestamp, - expectedStore.getLatestVersionPromoteToCurrentTimestamp()); - assertEquals(record.storeMetaValue.storeProperties.versions.size(), expectedStore.getVersions().size()); - assertEquals(record.storeMetaValue.storeProperties.systemStores.size(), expectedStore.getSystemStores().size()); + store.getLatestVersionPromoteToCurrentTimestamp()); + assertEquals(record.storeMetaValue.storeProperties.versions.size(), store.getVersions().size()); + assertEquals(record.storeMetaValue.storeProperties.systemStores.size(), store.getSystemStores().size()); + + // Store Key Schemas assertNotNull(record.storeMetaValue.storeKeySchemas); + assertNotNull(record.storeMetaValue.storeKeySchemas.keySchemaMap); + for (Map.Entry entry: record.storeMetaValue.storeKeySchemas.keySchemaMap.entrySet()) { + SchemaEntry expectedKeySchemaEntry = admin.getKeySchema(veniceCluster.getClusterName(), storeName); + + String actual = entry.getValue().toString(); + String expected = expectedKeySchemaEntry.getSchema().toString(); + assertEquals(actual, expected); + } + + // Store Value Schemas assertNotNull(record.storeMetaValue.storeValueSchemas); + for (Map.Entry entry: record.storeMetaValue.storeValueSchemas.valueSchemaMap + .entrySet()) { + int valueSchemaId = Integer.parseInt(entry.getKey().toString()); + SchemaEntry expectedValueSchemaEntry = + admin.getValueSchema(veniceCluster.getClusterName(), storeName, valueSchemaId); + + String actual = entry.getValue().toString(); + String expected = expectedValueSchemaEntry.getSchema().toString(); + assertEquals(actual, expected); + } assertNotNull(record.helixGroupInfo); assertNotNull(record.routingInfo); - - // Close - veniceServerWrapper.close(); - veniceCluster.close(); } }