From aee0f53b1b3933afbf23ef7e1b95b9cffa368af3 Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Tue, 26 Sep 2023 09:54:07 -0400 Subject: [PATCH 1/6] chore: simplified leaderboard data fetching function --- .../marginfi-v2-ui-state/src/lib/points.ts | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/marginfi-v2-ui-state/src/lib/points.ts b/packages/marginfi-v2-ui-state/src/lib/points.ts index 2898bd2551..e81e212f80 100644 --- a/packages/marginfi-v2-ui-state/src/lib/points.ts +++ b/packages/marginfi-v2-ui-state/src/lib/points.ts @@ -27,7 +27,55 @@ type LeaderboardRow = { socialPoints: number; }; -async function fetchLeaderboardData(connection?: Connection, rowCap = 100, pageSize = 50): Promise { +async function fetchLeaderboardData({ + connection, + queryCursor, + pageSize = 50, +}: { + connection?: Connection; + queryCursor?: string; + pageSize?: number; +}): Promise { + const pointsCollection = collection(firebaseApi.db, "points"); + + const pointsQuery: Query = query( + pointsCollection, + orderBy("total_points", "desc"), + ...(queryCursor ? [startAfter(queryCursor)] : []), + limit(pageSize) + ); + + const querySnapshot = await getDocs(pointsQuery); + const leaderboardSlice = querySnapshot.docs + .filter((item) => item.id !== null && item.id !== undefined && item.id != "None") + .map((doc) => { + const data = { id: doc.id, ...doc.data() } as LeaderboardRow; + return data; + }); + + const leaderboardFinalSlice: LeaderboardRow[] = [...leaderboardSlice]; + + if (connection) { + const publicKeys = leaderboardFinalSlice.map((value) => { + const [favoriteDomains] = FavouriteDomain.getKeySync(NAME_OFFERS_ID, new PublicKey(value.id)); + return favoriteDomains; + }); + const favoriteDomainsInfo = (await connection.getMultipleAccountsInfo(publicKeys)).map((accountInfo, idx) => + accountInfo ? FavouriteDomain.deserialize(accountInfo.data).nameAccount : publicKeys[idx] + ); + const reverseLookup = await reverseLookupBatch(connection, favoriteDomainsInfo); + + leaderboardFinalSlice.map((value, idx) => (value.id = reverseLookup[idx] ? `${reverseLookup[idx]}.sol` : value.id)); + } + + return leaderboardFinalSlice; +} + +async function fetchLeaderboardDataOriginal( + connection?: Connection, + rowCap = 100, + pageSize = 50 +): Promise { const pointsCollection = collection(firebaseApi.db, "points"); const leaderboardMap = new Map(); From 5dca603f34ffa481f9926326c9d4c09b3ff1b08f Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Tue, 26 Sep 2023 14:22:11 -0400 Subject: [PATCH 2/6] feat: use firebase cursor to paginate leaderboard table on scroll --- apps/marginfi-v2-ui/src/pages/points.tsx | 49 +++++++++++++++---- .../marginfi-v2-ui-state/src/lib/points.ts | 6 ++- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/apps/marginfi-v2-ui/src/pages/points.tsx b/apps/marginfi-v2-ui/src/pages/points.tsx index 77fa773f95..0ef04ff03d 100644 --- a/apps/marginfi-v2-ui/src/pages/points.tsx +++ b/apps/marginfi-v2-ui/src/pages/points.tsx @@ -1,4 +1,4 @@ -import { Fragment, useCallback, useMemo } from "react"; +import { Fragment, useCallback, useMemo, useRef } from "react"; import { Button, Table, @@ -34,6 +34,7 @@ import { numeralFormatter, groupedNumberFormatterDyn } from "@mrgnlabs/mrgn-comm import { useWalletContext } from "~/components/useWalletContext"; import { getFavoriteDomain } from "@bonfida/spl-name-service"; import { Connection, PublicKey } from "@solana/web3.js"; +import throttle from "lodash/throttle"; const HtmlTooltip = styled(({ className, ...props }: TooltipProps) => ( @@ -58,17 +59,22 @@ const Points: FC = () => { ]); const [leaderboardData, setLeaderboardData] = useState([]); + const [fetchingLeaderboardPage, setFetchingLeaderboardPage] = useState(false); + const fetchLeaderboardPage = useCallback(async () => { + const queryCursor = leaderboardData.length > 0 ? leaderboardData[leaderboardData.length - 1].doc : undefined; + fetchLeaderboardData({ + connection, + queryCursor, + }).then((data) => { + setLeaderboardData((current) => [...current, ...data]); + }); + }, [connection, leaderboardData, setLeaderboardData]); + const [domain, setDomain] = useState(); const currentUserId = useMemo(() => domain ?? currentFirebaseUser?.uid, [currentFirebaseUser, domain]); const referralCode = useMemo(() => routerQuery.referralCode as string | undefined, [routerQuery.referralCode]); - useEffect(() => { - if (connection && walletAddress) { - resolveDomain(connection, new PublicKey(walletAddress)); - } - }, [connection, walletAddress]); - const resolveDomain = async (connection: Connection, user: PublicKey) => { try { const { domain, reverse } = await getFavoriteDomain(connection, user); @@ -79,8 +85,33 @@ const Points: FC = () => { }; useEffect(() => { - fetchLeaderboardData(connection).then(setLeaderboardData); // TODO: cache leaderboard and avoid call - }, [connection, connected, walletAddress]); // Dependency array to re-fetch when these variables change + if (connection && walletAddress) { + resolveDomain(connection, new PublicKey(walletAddress)); + } + }, [connection, walletAddress]); + + useEffect(() => { + if (!leaderboardData.length) { + fetchLeaderboardData({ + connection, + }).then((data) => { + setLeaderboardData([...data]); + }); + } + + const handleScroll = throttle(() => { + if (fetchingLeaderboardPage) return; + setFetchingLeaderboardPage(false); + if (document.body.scrollTop > document.body.scrollHeight * 0.5) { + fetchLeaderboardPage(); + } + }, 500); + + document.body.addEventListener("scroll", handleScroll); + return () => { + document.body.removeEventListener("scroll", handleScroll); + }; + }, [connection, fetchLeaderboardPage, fetchingLeaderboardPage]); return ( <> diff --git a/packages/marginfi-v2-ui-state/src/lib/points.ts b/packages/marginfi-v2-ui-state/src/lib/points.ts index e81e212f80..69684f3f2c 100644 --- a/packages/marginfi-v2-ui-state/src/lib/points.ts +++ b/packages/marginfi-v2-ui-state/src/lib/points.ts @@ -11,6 +11,7 @@ import { getDoc, getCountFromServer, where, + QueryDocumentSnapshot, } from "firebase/firestore"; import { FavouriteDomain, NAME_OFFERS_ID, reverseLookupBatch } from "@bonfida/spl-name-service"; import { Connection, PublicKey } from "@solana/web3.js"; @@ -18,6 +19,7 @@ import { firebaseApi } from "."; type LeaderboardRow = { id: string; + doc: QueryDocumentSnapshot; total_activity_deposit_points: number; total_activity_borrow_points: number; total_referral_deposit_points: number; @@ -33,7 +35,7 @@ async function fetchLeaderboardData({ pageSize = 50, }: { connection?: Connection; - queryCursor?: string; + queryCursor?: QueryDocumentSnapshot; pageSize?: number; }): Promise { const pointsCollection = collection(firebaseApi.db, "points"); @@ -49,7 +51,7 @@ async function fetchLeaderboardData({ const leaderboardSlice = querySnapshot.docs .filter((item) => item.id !== null && item.id !== undefined && item.id != "None") .map((doc) => { - const data = { id: doc.id, ...doc.data() } as LeaderboardRow; + const data = { id: doc.id, doc, ...doc.data() } as LeaderboardRow; return data; }); From 5021beb5712a88b52950f9b3bc4c4350a15c8699 Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Tue, 26 Sep 2023 19:32:55 -0400 Subject: [PATCH 3/6] chore: remove original leaderboard data fetching function --- .../marginfi-v2-ui-state/src/lib/points.ts | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/packages/marginfi-v2-ui-state/src/lib/points.ts b/packages/marginfi-v2-ui-state/src/lib/points.ts index 69684f3f2c..e7c8e7a200 100644 --- a/packages/marginfi-v2-ui-state/src/lib/points.ts +++ b/packages/marginfi-v2-ui-state/src/lib/points.ts @@ -73,58 +73,6 @@ async function fetchLeaderboardData({ return leaderboardFinalSlice; } -async function fetchLeaderboardDataOriginal( - connection?: Connection, - rowCap = 100, - pageSize = 50 -): Promise { - const pointsCollection = collection(firebaseApi.db, "points"); - - const leaderboardMap = new Map(); - let initialQueryCursor = null; - do { - let pointsQuery: Query; - if (initialQueryCursor) { - pointsQuery = query( - pointsCollection, - orderBy("total_points", "desc"), - startAfter(initialQueryCursor), - limit(pageSize) - ); - } else { - pointsQuery = query(pointsCollection, orderBy("total_points", "desc"), limit(pageSize)); - } - - const querySnapshot = await getDocs(pointsQuery); - const leaderboardSlice = querySnapshot.docs.map((doc) => ({ id: doc.id, ...doc.data() })); - const leaderboardSliceFiltered = leaderboardSlice.filter( - (item) => item.id !== null && item.id !== undefined && item.id != "None" - ); - - for (const row of leaderboardSliceFiltered) { - leaderboardMap.set(row.id, row); - } - - initialQueryCursor = querySnapshot.docs.length > 0 ? querySnapshot.docs[querySnapshot.docs.length - 1] : null; - } while (initialQueryCursor !== null && leaderboardMap.size < rowCap); - - const leaderboardFinalSlice = [...leaderboardMap.values()].slice(0, 100); - - if (connection) { - const publicKeys = leaderboardFinalSlice.map((value) => { - const [favoriteDomains] = FavouriteDomain.getKeySync(NAME_OFFERS_ID, new PublicKey(value.id)); - return favoriteDomains; - }); - const favoriteDomainsInfo = (await connection.getMultipleAccountsInfo(publicKeys)).map((accountInfo, idx) => - accountInfo ? FavouriteDomain.deserialize(accountInfo.data).nameAccount : publicKeys[idx] - ); - const reverseLookup = await reverseLookupBatch(connection, favoriteDomainsInfo); - - leaderboardFinalSlice.map((value, idx) => (value.id = reverseLookup[idx] ? `${reverseLookup[idx]}.sol` : value.id)); - } - return leaderboardFinalSlice; -} - // Firebase query is very constrained, so we calculate the number of users with more points // as the the count of users with more points inclusive of corrupted rows - the count of corrupted rows async function fetchUserRank(userPoints: number): Promise { From bec362d25e27f66a6f5c43db41f1445ff459d921 Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Tue, 26 Sep 2023 20:03:12 -0400 Subject: [PATCH 4/6] chore: remove old throttle checks --- apps/marginfi-v2-ui/src/pages/points.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/marginfi-v2-ui/src/pages/points.tsx b/apps/marginfi-v2-ui/src/pages/points.tsx index 0ef04ff03d..71be989aae 100644 --- a/apps/marginfi-v2-ui/src/pages/points.tsx +++ b/apps/marginfi-v2-ui/src/pages/points.tsx @@ -59,7 +59,6 @@ const Points: FC = () => { ]); const [leaderboardData, setLeaderboardData] = useState([]); - const [fetchingLeaderboardPage, setFetchingLeaderboardPage] = useState(false); const fetchLeaderboardPage = useCallback(async () => { const queryCursor = leaderboardData.length > 0 ? leaderboardData[leaderboardData.length - 1].doc : undefined; fetchLeaderboardData({ @@ -100,8 +99,6 @@ const Points: FC = () => { } const handleScroll = throttle(() => { - if (fetchingLeaderboardPage) return; - setFetchingLeaderboardPage(false); if (document.body.scrollTop > document.body.scrollHeight * 0.5) { fetchLeaderboardPage(); } @@ -111,7 +108,7 @@ const Points: FC = () => { return () => { document.body.removeEventListener("scroll", handleScroll); }; - }, [connection, fetchLeaderboardPage, fetchingLeaderboardPage]); + }, [connection, fetchLeaderboardPage]); return ( <> From e2d41884b05c42ceab69a27c17ec5c6434df600b Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Wed, 27 Sep 2023 12:46:14 -0400 Subject: [PATCH 5/6] feat: add skeleton loaders on initial load --- apps/marginfi-v2-ui/src/pages/points.tsx | 192 +++++++++++++---------- 1 file changed, 106 insertions(+), 86 deletions(-) diff --git a/apps/marginfi-v2-ui/src/pages/points.tsx b/apps/marginfi-v2-ui/src/pages/points.tsx index 71be989aae..0a0640ced9 100644 --- a/apps/marginfi-v2-ui/src/pages/points.tsx +++ b/apps/marginfi-v2-ui/src/pages/points.tsx @@ -58,14 +58,16 @@ const Points: FC = () => { state.userPointsData, ]); - const [leaderboardData, setLeaderboardData] = useState([]); + const [leaderboardData, setLeaderboardData] = useState([...new Array(50).fill({})]); const fetchLeaderboardPage = useCallback(async () => { const queryCursor = leaderboardData.length > 0 ? leaderboardData[leaderboardData.length - 1].doc : undefined; + setLeaderboardData((current) => [...current, ...new Array(50).fill({})]); fetchLeaderboardData({ connection, queryCursor, }).then((data) => { - setLeaderboardData((current) => [...current, ...data]); + const filtered = [...leaderboardData].filter((row) => row.hasOwnProperty("id")); + setLeaderboardData((current) => [...filtered, ...data]); }); }, [connection, leaderboardData, setLeaderboardData]); @@ -90,7 +92,7 @@ const Points: FC = () => { }, [connection, walletAddress]); useEffect(() => { - if (!leaderboardData.length) { + if (!leaderboardData[0].hasOwnProperty("id")) { fetchLeaderboardData({ connection, }).then((data) => { @@ -361,90 +363,108 @@ const Points: FC = () => { - {leaderboardData.map((row: LeaderboardRow, index: number) => ( - - - {index === 0 ? "🥇" : index === 1 ? "🥈" : index === 2 ? "🥉" : index + 1} - - - { + if (!row.hasOwnProperty("id")) { + return ( + + {[...new Array(7)].map((_, index) => ( + + + + ))} + + ); + } + + const data = row as LeaderboardRow; + + return ( + + - {row.id.endsWith(".sol") ? row.id : `${row.id.slice(0, 5)}...${row.id.slice(-5)}`} - - - - - {groupedNumberFormatterDyn.format(Math.round(row.total_activity_deposit_points))} - - - {groupedNumberFormatterDyn.format(Math.round(row.total_activity_borrow_points))} - - - {groupedNumberFormatterDyn.format( - Math.round(row.total_referral_deposit_points + row.total_referral_borrow_points) - )} - - - {groupedNumberFormatterDyn.format(Math.round(row.socialPoints ? row.socialPoints : 0))} - - - {groupedNumberFormatterDyn.format( - Math.round( - row.total_deposit_points + row.total_borrow_points + (row.socialPoints ? row.socialPoints : 0) - ) - )} - - - ))} + {index === 0 ? "🥇" : index === 1 ? "🥈" : index === 2 ? "🥉" : index + 1} + + + + {data.id.endsWith(".sol") ? data.id : `${data.id.slice(0, 5)}...${data.id.slice(-5)}`} + + + + + {groupedNumberFormatterDyn.format(Math.round(data.total_activity_deposit_points))} + + + {groupedNumberFormatterDyn.format(Math.round(data.total_activity_borrow_points))} + + + {groupedNumberFormatterDyn.format( + Math.round(data.total_referral_deposit_points + data.total_referral_borrow_points) + )} + + + {groupedNumberFormatterDyn.format(Math.round(data.socialPoints ? data.socialPoints : 0))} + + + {groupedNumberFormatterDyn.format( + Math.round( + data.total_deposit_points + + data.total_borrow_points + + (data.socialPoints ? data.socialPoints : 0) + ) + )} + + + ); + })} From eaab2ebea14742d362bfded7d4f77973013197a2 Mon Sep 17 00:00:00 2001 From: Adam Chambers Date: Wed, 27 Sep 2023 14:10:22 -0400 Subject: [PATCH 6/6] feat: improved leaderboard pagination, switch to sentinel intersection observer / current page in state --- apps/marginfi-v2-ui/src/pages/points.tsx | 76 ++++++++++++++++++++---- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/apps/marginfi-v2-ui/src/pages/points.tsx b/apps/marginfi-v2-ui/src/pages/points.tsx index 0a0640ced9..3d9cffca4b 100644 --- a/apps/marginfi-v2-ui/src/pages/points.tsx +++ b/apps/marginfi-v2-ui/src/pages/points.tsx @@ -58,18 +58,49 @@ const Points: FC = () => { state.userPointsData, ]); - const [leaderboardData, setLeaderboardData] = useState([...new Array(50).fill({})]); + const leaderboardPerPage = 50; + const [leaderboardData, setLeaderboardData] = useState([ + ...new Array(leaderboardPerPage).fill({}), + ]); + const [leaderboardPage, setLeaderboardPage] = useState(0); + const [isFetchingLeaderboardPage, setIsFetchingLeaderboardPage] = useState(false); + const leaderboardSentinelRef = useRef(null); + + // fetch next page of leaderboard results const fetchLeaderboardPage = useCallback(async () => { - const queryCursor = leaderboardData.length > 0 ? leaderboardData[leaderboardData.length - 1].doc : undefined; + // grab last row of current leaderboard data for cursor + const lastRow = [...leaderboardData].filter((row) => row.hasOwnProperty("id"))[ + leaderboardPage * leaderboardPerPage - 2 + ] as LeaderboardRow; + if (!lastRow || !lastRow.hasOwnProperty("id")) return; + setIsFetchingLeaderboardPage(true); + + // fetch new page of data with cursor + const queryCursor = leaderboardData.length > 0 ? lastRow.doc : undefined; setLeaderboardData((current) => [...current, ...new Array(50).fill({})]); fetchLeaderboardData({ connection, queryCursor, }).then((data) => { + // filter out skeleton rows const filtered = [...leaderboardData].filter((row) => row.hasOwnProperty("id")); - setLeaderboardData((current) => [...filtered, ...data]); + + // additional check for duplicate values + const uniqueData = data.reduce((acc, curr) => { + const isDuplicate = acc.some((item) => { + const data = item as LeaderboardRow; + data.id === curr.id; + }); + if (!isDuplicate) { + acc.push(curr); + } + return acc; + }, filtered); + + setLeaderboardData(uniqueData); + setIsFetchingLeaderboardPage(false); }); - }, [connection, leaderboardData, setLeaderboardData]); + }, [connection, leaderboardData, setLeaderboardData, setIsFetchingLeaderboardPage, leaderboardPage]); const [domain, setDomain] = useState(); @@ -91,8 +122,14 @@ const Points: FC = () => { } }, [connection, walletAddress]); + // fetch new page when page counter changed useEffect(() => { - if (!leaderboardData[0].hasOwnProperty("id")) { + fetchLeaderboardPage(); + }, [leaderboardPage]); + + useEffect(() => { + // fetch initial page and overwrite skeleton rows + if (leaderboardPage === 0) { fetchLeaderboardData({ connection, }).then((data) => { @@ -100,15 +137,29 @@ const Points: FC = () => { }); } - const handleScroll = throttle(() => { - if (document.body.scrollTop > document.body.scrollHeight * 0.5) { - fetchLeaderboardPage(); + // intersection observer to fetch new page of data + // when sentinel element is scrolled into view + const observer = new IntersectionObserver( + (entries) => { + if (entries[0].isIntersecting && !isFetchingLeaderboardPage) { + setLeaderboardPage((current) => current + 1); + } + }, + { + root: null, + rootMargin: "0px", + threshold: 0, } - }, 500); + ); + + if (leaderboardSentinelRef.current) { + observer.observe(leaderboardSentinelRef.current); + } - document.body.addEventListener("scroll", handleScroll); return () => { - document.body.removeEventListener("scroll", handleScroll); + if (leaderboardSentinelRef.current) { + observer.unobserve(leaderboardSentinelRef.current); + } }; }, [connection, fetchLeaderboardPage]); @@ -395,7 +446,7 @@ const Points: FC = () => { style={{ fontWeight: 400 }} > { +
);