Feat/relations indexing cleanup

src/fetching/fetchAllowlistFromUri.ts

@@ -1,5 +1,6 @@
 import { fetchFromHTTPS, fetchFromIPFS } from "@/utils";
 import { StandardMerkleTree } from "@openzeppelin/merkle-tree";
+import { fetchFromHttpsOrIpfs } from "@/utils/fetchFromHttpsOrIpfs";
 
 /*
  * This function fetches the metadata of a claim from the uri as stored in the claim on the contract.
@@ -29,27 +30,7 @@ interface FetchAllowListFromUri {
 }
 
 export const fetchAllowListFromUri = async ({ uri }: FetchAllowListFromUri) => {
-  if (!uri || uri === "ipfs://null" || uri === "ipfs://") {
-    console.error("[FetchAllowListFromUri] URI is missing: ", uri);
-    return;
-  }
-
-  let fetchResult;
-
-  // Try from IPFS
-  if (uri.startsWith("ipfs://")) {
-    fetchResult = await fetchFromIPFS({ uri });
-  }
-
-  // Try from HTTPS
-  if (uri.startsWith("https://")) {
-    fetchResult = await fetchFromHTTPS({ uri });
-  }
-
-  // If nothing found yet, try from IPFS as CID
-  if (!fetchResult) {
-    fetchResult = await fetchFromIPFS({ uri });
-  }
+  const fetchResult = await fetchFromHttpsOrIpfs(uri);
 
   if (!fetchResult) {
     console.error(
@@ -60,12 +41,30 @@ export const fetchAllowListFromUri = async ({ uri }: FetchAllowListFromUri) => {
 
   // If response object is already a OZ Merkle tree, return it as is
   try {
-    return StandardMerkleTree.load<[string, bigint]>(JSON.parse(fetchResult));
+    console.debug(
+      "[FetchAllowListFromUri] Loading OZ Merkle tree from response by parsing as JSON",
+    );
+
+    return StandardMerkleTree.load<[string, bigint]>(
+      JSON.parse(fetchResult as string),
+    );
+  } catch (error) {
+    console.error(
+      `[FetchAllowListFromUri] Allow list at ${uri} is not a valid OZ Merkle tree`,
+      error,
+    );
+  }
+
+  // If response object is already a OZ Merkle tree, return it as is
+  try {
+    console.debug(
+      "[FetchAllowListFromUri] Loading OZ Merkle tree directly from response",
+    );
+    return StandardMerkleTree.load<[string, bigint]>(fetchResult as never);
   } catch (error) {
     console.error(
       `[FetchAllowListFromUri] Allow list at ${uri} is not a valid OZ Merkle tree`,
       error,
     );
-    return;
   }
 };

src/fetching/fetchMetadataFromUri.ts

@@ -1,6 +1,6 @@
-import { validateMetaData } from "@hypercerts-org/sdk";
-import { fetchFromHTTPS, fetchFromIPFS } from "@/utils";
+import { HypercertMetadata, validateMetaData } from "@hypercerts-org/sdk";
 import { Tables } from "@/types/database.types";
+import { fetchFromHttpsOrIpfs } from "@/utils/fetchFromHttpsOrIpfs";
 
 /*
  * This function fetches the metadata of a claim from the uri as stored in the claim on the contract.
@@ -30,27 +30,7 @@ interface FetchMetadataFromUri {
 }
 
 export const fetchMetadataFromUri = async ({ uri }: FetchMetadataFromUri) => {
-  if (!uri || uri === "ipfs://null" || uri === "ipfs://") {
-    console.error("[FetchMetadataFromUri] URI is missing");
-    return;
-  }
-
-  let fetchResult;
-
-  // Try from IPFS
-  if (uri.startsWith("ipfs://")) {
-    fetchResult = await fetchFromIPFS({ uri });
-  }
-
-  // Try from HTTPS
-  if (uri.startsWith("https://")) {
-    fetchResult = await fetchFromHTTPS({ uri });
-  }
-
-  // If nothing found yet, try from IPFS as CID
-  if (!fetchResult) {
-    fetchResult = await fetchFromIPFS({ uri });
-  }
+  const fetchResult = await fetchFromHttpsOrIpfs(uri);
 
   if (!fetchResult) {
     console.error(
@@ -59,31 +39,33 @@ export const fetchMetadataFromUri = async ({ uri }: FetchMetadataFromUri) => {
     return;
   }
 
-  const validation = validateMetaData(fetchResult);
+  const { valid, data, errors } = validateMetaData(fetchResult);
 
-  if (!validation.valid) {
+  if (!valid) {
     console.error(
       `[FetchMetadataFromUri] Invalid metadata for URI ${uri}`,
-      validation.errors,
+      errors,
     );
     return;
   }
 
+  const _metadata = data as HypercertMetadata;
+
   const metadata: Partial<Tables<"metadata">> = {
-    name: fetchResult.name,
-    description: fetchResult.description,
-    external_url: fetchResult.external_url,
-    image: fetchResult.image,
+    name: _metadata.name,
+    description: _metadata.description,
+    external_url: _metadata.external_url,
+    image: _metadata.image,
     properties: fetchResult.properties,
-    contributors: fetchResult.hypercert.contributors.value,
-    impact_scope: fetchResult.hypercert.impact_scope.value,
-    impact_timeframe_from: fetchResult.hypercert.impact_timeframe.value[0],
-    impact_timeframe_to: fetchResult.hypercert.impact_timeframe.value[1],
-    work_scope: fetchResult.hypercert.work_scope.value,
-    work_timeframe_from: fetchResult.hypercert.work_timeframe.value[0],
-    work_timeframe_to: fetchResult.hypercert.work_timeframe.value[1],
-    rights: fetchResult.hypercert.rights.value,
-    allow_list_uri: fetchResult.allowList,
+    contributors: _metadata.hypercert?.contributors.value,
+    impact_scope: _metadata.hypercert?.impact_scope.value,
+    impact_timeframe_from: _metadata.hypercert?.impact_timeframe?.value?.[0],
+    impact_timeframe_to: _metadata.hypercert?.impact_timeframe?.value?.[1],
+    work_scope: _metadata.hypercert?.work_scope.value,
+    work_timeframe_from: _metadata.hypercert?.work_timeframe?.value?.[0],
+    work_timeframe_to: _metadata.hypercert?.work_timeframe?.value?.[1],
+    rights: _metadata.hypercert?.rights?.value,
+    allow_list_uri: _metadata.allowList,
   };
 
   return metadata;

src/indexer/indexAllowlistCreated.ts

@@ -66,14 +66,14 @@ export const indexAllowListCreated = async ({
 
       const allowListData = allowLists.map((allowList) => ({
         token_id: allowList.token_id.toString(),
-        contract_id: contractEvent.contract_id,
+        contract_id: contractEvent.contracts_id,
         root: allowList.root,
       }));
 
       return {
         allowListData,
         contractEventUpdate: {
-          id: contractEvent.id,
+          ...contractEvent,
           last_block_indexed: toBlock,
         },
       };

src/indexer/indexAllowlistData.ts

@@ -1,8 +1,9 @@
 import { IndexerConfig } from "@/types/types";
-import { getMissingAllowListUris } from "@/storage/getMissingAllowListUris";
+import { getIncompleteAllowLists } from "@/storage/getIncompleteAllowLists";
 import { fetchAllowListFromUri } from "@/fetching/fetchAllowlistFromUri";
 import { storeAllowListData } from "@/storage/storeAllowListData";
-import { Database, Tables } from "@/types/database.types";
+import { Tables } from "@/types/database.types";
+import { getUnparsedAllowLists } from "@/storage/getUnparsedAllowLists";
 
 /*
  * This function indexes the logs of the ClaimStored event emitted by the HypercertMinter contract. Based on the last
@@ -24,7 +25,7 @@ const defaultConfig = {
 export const indexAllowListData = async ({
   batchSize = defaultConfig.batchSize,
 }: IndexerConfig = defaultConfig) => {
-  const missingAllowLists = await getMissingAllowListUris();
+  const missingAllowLists = await getUnparsedAllowLists();
 
   if (!missingAllowLists || missingAllowLists.length === 0) {
     console.debug("[IndexAllowListData] No missing allow lists found");
@@ -46,31 +47,32 @@ export const indexAllowListData = async ({
 };
 
 const processAllowListBatch = async (
-  batch: Database["public"]["Functions"]["find_missing_allow_list_uris_and_roots"]["Returns"],
+  batch: Partial<Tables<"allow_list_data">>[],
 ) => {
   const allowListData = (
     await Promise.all(
       batch.map(async (missingList) => {
+        if (!missingList.uri) {
+          console.error(
+            `[IndexAllowListData] Missing URI for allow list ${missingList.id}`,
+          );
+          return;
+        }
+
         const allowList = await fetchAllowListFromUri({
-          uri: missingList.allow_list_uri,
+          uri: missingList.uri,
         });
 
         if (!allowList) {
           return;
         }
 
-        if (missingList.allow_list_root !== allowList.root) {
-          console.error(
-            `[IndexAllowListData] Root hash mismatch for allow list ${missingList.allow_list_uri}: expected ${missingList.allow_list_root}, got ${allowList.root}`,
-          );
-          return;
-        }
-
         return {
-          id: missingList.allow_list_id,
+          id: missingList.id,
           data: JSON.stringify(allowList.dump()),
-          root: missingList.allow_list_root,
-          uri: missingList.allow_list_uri,
+          root: allowList.root,
+          uri: missingList.uri,
+          parsed: true,
         } as Tables<"allow_list_data">;
       }),
     )

src/indexer/indexAllowlistRecords.ts

@@ -1,12 +1,11 @@
 import { IndexerConfig } from "@/types/types";
-import { storeAllowListData } from "@/storage/storeAllowListData";
-import {
-  AllowList,
-  getUnparsedAllowLists,
-} from "@/storage/getUnparsedAllowLists";
-import { StandardMerkleTree } from "@openzeppelin/merkle-tree";
 import { storeAllowListRecords } from "@/storage/storeAllowListRecords";
 import { Tables } from "@/types/database.types";
+import {
+  getUnparsedAllowListRecords,
+  UnparsedAllowListRecord,
+} from "@/storage/getUnparsedAllowListsRecords";
+import { StandardMerkleTree } from "@openzeppelin/merkle-tree";
 
 /*
  * This function indexes the logs of the ClaimStored event emitted by the HypercertMinter contract. Based on the last
@@ -25,13 +24,16 @@ const defaultConfig = {
   batchSize: 2n,
 };
 
+//TODO allow list records parsing based on created allowlists and claims
 export const indexAllowlistRecords = async ({
   batchSize = defaultConfig.batchSize,
 }: IndexerConfig = defaultConfig) => {
-  const unparsedAllowLists = await getUnparsedAllowLists();
+  const unparsedAllowLists = await getUnparsedAllowListRecords();
 
   if (!unparsedAllowLists || unparsedAllowLists.length === 0) {
-    console.debug("[IndexAllowlistRecords] No unparsed allow lists found");
+    console.debug(
+      "[IndexAllowlistRecords] No parsable unparsed allow lists found",
+    );
     return;
   }
 
@@ -50,27 +52,12 @@ export const indexAllowlistRecords = async ({
   }
 };
 
-const processAllowListEntriesBatch = async (batch: AllowList[]) => {
-  const rows = await Promise.all(
+const processAllowListEntriesBatch = async (
+  batch: UnparsedAllowListRecord[],
+) => {
+  const allowListsToStore = await Promise.all(
     batch.map(async (allowList) => {
-      //TODO fix typing of data
-      if (!allowList.allow_list_data) {
-        console.warn(
-          `[IndexAllowlistRecords] Missing data for allow list ${allowList.id}`,
-          allowList,
-        );
-        return;
-      } else if (!allowList.allow_list_data.data) {
-        console.warn(
-          `[IndexAllowlistRecords] Missing data for allow list ${allowList.id}`,
-          allowList,
-        );
-        return;
-      }
-
-      const tree = StandardMerkleTree.load(
-        JSON.parse(<string>allowList.allow_list_data.data),
-      );
+      const tree = StandardMerkleTree.load(JSON.parse(allowList.data));
 
       if (!tree) {
         console.error(
@@ -83,25 +70,26 @@ const processAllowListEntriesBatch = async (batch: AllowList[]) => {
       const rows = [];
       for (const [i, v] of tree.entries()) {
         rows.push({
-          hc_allow_list_id: allowList.id,
           user_address: v[0],
           entry: i,
           units: v[1],
         });
       }
 
-      return rows;
+      return { ...allowList, records: rows };
     }),
   );
 
-  const allowListRecords = rows
-    .flat()
-    .filter(
-      (r): r is Tables<"allow_list_records"> => r !== null && r !== undefined,
-    );
-
   try {
-    await storeAllowListRecords({ allowListRecords });
+    await Promise.all(
+      allowListsToStore.map((data) =>
+        storeAllowListRecords({
+          claim_id: data?.claim_id,
+          allow_list_data_id: data?.al_data_id,
+          records: data?.records,
+        }),
+      ),
+    );
   } catch (error) {
     console.error(
       "[IndexAllowlistRecords] Error while storing allow list records",

src/indexer/indexClaimsStored.ts

@@ -20,7 +20,7 @@ import { storeClaim } from "@/storage/storeClaim";
  */
 
 const defaultConfig = {
-  batchSize: 10000n,
+  batchSize: 20000n,
   eventName: "ClaimStored",
 };
 
@@ -50,6 +50,10 @@ export const indexClaimsStoredEvents = async ({
       });
 
       if (!logsFound) {
+        console.debug(
+          " [IndexClaimsStored] No logs found for contract event",
+          contractEvent,
+        );
         return;
       }
 
@@ -65,13 +69,13 @@ export const indexClaimsStoredEvents = async ({
 
       const claims = parsedEvents.map((claim) => ({
         ...claim,
-        contract_id: contractEvent.contract_id,
+        contracts_id: contractEvent.contracts_id,
       }));
 
       return {
         claims,
         contractEventUpdate: {
-          id: contractEvent.id,
+          ...contractEvent,
           last_block_indexed: toBlock,
         },
       };