From 6b832f17ea7efcd2b84cef9c008d31f78af70aef Mon Sep 17 00:00:00 2001 From: Dirk Holtwick Date: Sat, 5 Oct 2024 23:37:27 +0200 Subject: [PATCH 1/3] feat: lazy load chunk wise --- lib/basic/lazy-data.spec.ts | 97 ++++++++++++++++++++++++++++++++++++- lib/basic/lazy-data.ts | 57 ++++++++++++++++++---- package.json | 8 +-- 3 files changed, 148 insertions(+), 14 deletions(-) diff --git a/lib/basic/lazy-data.spec.ts b/lib/basic/lazy-data.spec.ts index 2648e42..ff953e8 100644 --- a/lib/basic/lazy-data.spec.ts +++ b/lib/basic/lazy-data.spec.ts @@ -1,9 +1,13 @@ +import type { LoggerInterface } from 'zeed' import { describe, expect, it } from 'vitest' -import { createArray } from 'zeed' +import { createArray, Logger, sleep } from 'zeed' import { useLazyData } from './lazy-data' +const log: LoggerInterface = Logger('lazy-data.spec') + describe('useLazyData', () => { async function onFetch(offset: number, length: number) { + log('onFetch', offset, length) return createArray(length).map((_, i) => offset + i) } @@ -49,4 +53,95 @@ describe('useLazyData', () => { expect(data.length).toBe(3) expect(data.every(item => item == null)).toBe(true) }) + + it('should set visible data correctly', async () => { + const { setSize, setVisible, data } = useLazyData({ + onFetch, + chunkSize: 2, + margin: 2, + }) + setSize(20) + const r = setVisible(3, 5) + expect(r).toMatchInlineSnapshot(` + { + "allChunksToLoad": [ + 0, + 1, + 2, + 3, + ], + "fromChunk": 0, + "fromIndex": 1, + "toChunk": 3, + "toIndex": 7, + } + `) + + await sleep(100) + + expect(data).toMatchInlineSnapshot(` + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ] + `) + + const r2 = setVisible(4, 6) + expect(r2).toMatchInlineSnapshot(` + { + "allChunksToLoad": [ + 4, + ], + "fromChunk": 1, + "fromIndex": 2, + "toChunk": 4, + "toIndex": 8, + } + `) + + await sleep(100) + + expect(data).toMatchInlineSnapshot(` + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + ] + `) + }) }) diff --git a/lib/basic/lazy-data.ts b/lib/basic/lazy-data.ts index c0740ac..6bbb592 100644 --- a/lib/basic/lazy-data.ts +++ b/lib/basic/lazy-data.ts @@ -42,17 +42,56 @@ export function useLazyData(opt: Config) { function setVisible(fromPos: number, toPos: number) { log.assert(fromPos <= toPos, 'fromPos must be less than or equal to toPos') - const fromChunk = Math.floor(Math.max(0, fromPos - margin) / chunkSize) - const toChunk = Math.floor((toPos + margin) / chunkSize) * chunkSize - const iterationNow = iteration - onFetch(fromChunk * chunkSize, ((toChunk - fromChunk) * chunkSize) + chunkSize).then((result) => { - if (iterationNow !== iteration) + const fromIndex = Math.max(0, fromPos - margin) + const toIndex = Math.min(dataSize, toPos + margin) + const fromChunk = Math.floor(fromIndex / chunkSize) + const toChunk = Math.floor(toIndex / chunkSize) + + const allChunksToLoad: number[] = [] + + let chunksToLoad: number[] = [] + + function flush() { + if (chunksToLoad.length === 0) return - for (let i = fromChunk; i <= toChunk; i++) - chunks[i] = ChunkStatus.Loaded + const loading = [...chunksToLoad] + allChunksToLoad.push(...loading) + chunksToLoad = [] + + const fromIndex = loading[0] * chunkSize + const itemCount = (loading[loading.length - 1] - loading[0] + 1) * chunkSize + + onFetch(fromIndex, itemCount).then((values) => { + log('Loaded', fromIndex, itemCount, values) + loading.forEach(i => chunks[i] = ChunkStatus.Loaded) + for (let i = 0; i < itemCount; i++) { + data[fromIndex + i] = values[i] as any + } + }).catch((e) => { + log.error('Failed to load chunks', loading, e) + }) + } + + for (let i = fromChunk; i <= toChunk; i++) { + const chunk = chunks[i] + if (chunk == null) { + chunks[i] = ChunkStatus.Loading + chunksToLoad.push(i) + } + else { + flush() + } + } + + flush() - // arraySetArrayInPlace(data, result, fromChunk * chunkSize) - }) + return { + fromIndex, + toIndex, + fromChunk, + toChunk, + allChunksToLoad, + } } return { diff --git a/package.json b/package.json index 2ffe847..4bc301e 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "build": "vite build", "build:demo": "BUILD_DEMO=1 vite build", "check": "vue-tsc --noEmit", - "dev": "vite --host", + "dev": "vite --host --open", "generate-pwa-assets": "npx pwa-assets-generator", "lint": "eslint .", "lint:fix": "eslint . --fix", @@ -75,7 +75,7 @@ "dependencies": { "@floating-ui/vue": "^1.1.5", "@vueuse/core": "^11.1.0", - "vue": "^3.5.10", + "vue": "^3.5.11", "zeed": "^0.24.22" }, "devDependencies": { @@ -83,7 +83,7 @@ "@antfu/ni": "^0.23.0", "@shikijs/markdown-it": "^1.21.0", "@vitejs/plugin-vue": "^5.1.4", - "@vitest/browser": "^2.1.1", + "@vitest/browser": "^2.1.2", "eslint": "^9", "stylus": "^0.63.0", "tsup": "^8.3.0", @@ -92,6 +92,6 @@ "vite": "^5.4.8", "vite-plugin-dts": "^4.2.3", "vite-plugin-qrcode": "^0.2.3", - "vitest": "^2.1.1" + "vitest": "^2.1.2" } } From 4eacdc9cf1d7ca4828347ce79250879ca2f2feb9 Mon Sep 17 00:00:00 2001 From: Dirk Holtwick Date: Sat, 5 Oct 2024 23:48:11 +0200 Subject: [PATCH 2/3] feat: works with virtual list --- lib/basic/lazy-data.spec.ts | 4 ++-- lib/basic/lazy-data.ts | 9 ++++++--- lib/basic/oui-virtual-list.demo.vue | 25 ++++++++++++++++++++----- lib/basic/oui-virtual-list.vue | 10 ++++++---- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/basic/lazy-data.spec.ts b/lib/basic/lazy-data.spec.ts index ff953e8..dc3f48f 100644 --- a/lib/basic/lazy-data.spec.ts +++ b/lib/basic/lazy-data.spec.ts @@ -61,7 +61,7 @@ describe('useLazyData', () => { margin: 2, }) setSize(20) - const r = setVisible(3, 5) + const r = setVisible(3, 2) expect(r).toMatchInlineSnapshot(` { "allChunksToLoad": [ @@ -104,7 +104,7 @@ describe('useLazyData', () => { ] `) - const r2 = setVisible(4, 6) + const r2 = setVisible(4, 2) expect(r2).toMatchInlineSnapshot(` { "allChunksToLoad": [ diff --git a/lib/basic/lazy-data.ts b/lib/basic/lazy-data.ts index 6bbb592..7baf13b 100644 --- a/lib/basic/lazy-data.ts +++ b/lib/basic/lazy-data.ts @@ -1,3 +1,4 @@ +import type { Reactive } from 'vue' import type { LoggerInterface } from 'zeed' import { reactive } from 'vue' import { arrayEmptyInPlace, createArray, Logger } from 'zeed' @@ -9,6 +10,7 @@ interface Config { chunkSize?: number margin?: number size?: number + data?: Reactive } enum ChunkStatus { @@ -22,7 +24,7 @@ export function useLazyData(opt: Config) { let iteration = 0 let dataSize = 0 - const data = reactive<(T | null)[]>([]) + const data = opt.data ?? reactive<(T | null)[]>([]) const chunks: Record = {} function reload() { @@ -40,8 +42,9 @@ export function useLazyData(opt: Config) { if (size > 0) setSize(size) - function setVisible(fromPos: number, toPos: number) { - log.assert(fromPos <= toPos, 'fromPos must be less than or equal to toPos') + function setVisible(fromPos: number, length: number) { + const toPos = fromPos + length + // log.assert(fromPos <= toPos, 'fromPos must be less than or equal to toPos') const fromIndex = Math.max(0, fromPos - margin) const toIndex = Math.min(dataSize, toPos + margin) const fromChunk = Math.floor(fromIndex / chunkSize) diff --git a/lib/basic/oui-virtual-list.demo.vue b/lib/basic/oui-virtual-list.demo.vue index a2dd9ed..36774b4 100644 --- a/lib/basic/oui-virtual-list.demo.vue +++ b/lib/basic/oui-virtual-list.demo.vue @@ -1,19 +1,34 @@