From 3f6bfeff7e80b5bb3fe5993daa69dd0e361c1894 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 25 Jul 2024 12:14:11 +0800 Subject: [PATCH 1/5] winget-source: implement multi-chunk MSZIP decompression --- winget-source/utilities.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index b47ae97..9d66a3e 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -87,10 +87,20 @@ function getRemoteURL(uri) { * @returns {Buffer} The decompressed buffer. */ async function decompressMSZIP(buffer) { - if (buffer.toString('ascii', 28, 30) != 'CK') { + const magicHeader = Buffer.from([0, 0, 0x43, 0x4b]); + if (!buffer.subarray(26, 30).equals(magicHeader)) { throw new Error('Invalid MSZIP format'); } - return await inflateRaw(buffer.subarray(30)); + var chunkIndex = 26; + var decompressed = Buffer.alloc(0); + while ((chunkIndex = buffer.indexOf(magicHeader, chunkIndex)) > -1) { + chunkIndex += magicHeader.byteLength; + const decompressedChunk = await inflateRaw(buffer.subarray(chunkIndex), { + dictionary: decompressed.subarray(-32768) + }); + decompressed = Buffer.concat([decompressed, decompressedChunk]); + } + return decompressed; } /** From be9a3757b437bbf2111fc473abcca9270c4c83c1 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 25 Jul 2024 12:38:36 +0800 Subject: [PATCH 2/5] winget-source: write package metadata to disk only after manifests are synced -- a naive implementation --- winget-source/sync-repo.js | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 5f205da..243bfed 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -34,12 +34,11 @@ async function syncIndex(version, handler) { const sourceFilename = version > 1 ? `source${version}.msix` : 'source.msix'; try { // download index package to buffer - const [indexBuffer, modifiedDate, updated] = await syncFile(sourceFilename, true, false); - if (!updated) { + const [indexBuffer, modifiedDate] = await syncFile(sourceFilename, true, false); + if (!indexBuffer) { winston.info(`skip syncing version ${version} from ${remote}`); return; } - assert(indexBuffer !== null, "Failed to get the source index buffer!"); // unpack, extract and load index database try { @@ -76,17 +75,15 @@ await syncIndex(2, async (db) => { try { const packageURIs = buildPackageMetadataURIs(await db.all('SELECT id, hash FROM packages')); try { - // sync latest package metadata in parallel - const manifestURIs = await async.concatLimit(packageURIs, parallelLimit, async (uri) => { - const [metadataBuffer] = await syncFile(uri, false); - try { - return metadataBuffer ? await buildManifestURIsFromPackageMetadata(metadataBuffer) : []; - } catch (error) { - exitWithCode(EX_SOFTWARE, error); + // sync latest package metadata and manifests in parallel + await async.eachLimit(packageURIs, parallelLimit, async (uri) => { + const [metadataBuffer, modifiedDate] = await syncFile(uri, false, false); + if (metadataBuffer) { + const manifestURIs = await buildManifestURIsFromPackageMetadata(metadataBuffer); + await async.eachSeries(manifestURIs, async (uri) => await syncFile(uri, false)); + await cacheFileWithURI(uri, metadataBuffer, modifiedDate); } }); - // sync latest manifests in parallel - await async.eachLimit(manifestURIs, parallelLimit, async (uri) => await syncFile(uri, false)); } catch (error) { exitWithCode(EX_TEMPFAIL, error); } From 0787ab6dbd4b32b9e91a09fc918d19f3fec9c77c Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 25 Jul 2024 12:55:43 +0800 Subject: [PATCH 3/5] winget-source: add support for force sync --- README.md | 7 ++++--- winget-source/sync-repo.js | 13 +++++++------ winget-source/sync.sh | 1 + winget-source/utilities.js | 6 +++++- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f2dd1e8..9bd18a8 100644 --- a/README.md +++ b/README.md @@ -443,10 +443,11 @@ An alternative HTTP(S) syncing tool, replacing `rclone` and `lftp` in some cases A handy tool to sync pre-indexed [Windows Package Manager](https://github.com/microsoft/winget-cli) (aka. WinGet) sources. -| Parameter | Description | -| --------------------- | ---------------------------------------------------------------- | +| Parameter | Description | +| --------------------- | ----------------------------------------------------------------- | +| `WINGET_FORCE_SYNC` | Force syncs everything against the upstream. Defaults to `false`. | | `WINGET_REPO_URL` | Sets the URL of upstream. Defaults to [`https://cdn.winget.microsoft.com/cache`](https://cdn.winget.microsoft.com/cache) | -| `WINGET_REPO_JOBS` | Parallel jobs. Defaults to 8. | +| `WINGET_REPO_JOBS` | Parallel jobs. Defaults to 8. | ### yukina diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 243bfed..3185ccf 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -19,7 +19,7 @@ import { } from './utilities.js' -const { parallelLimit, remote, sqlite3, winston } = setupEnvironment(); +const { forceSync, parallelLimit, remote, sqlite3, winston } = setupEnvironment(); /** * Sync with the official WinGet repository index. @@ -34,11 +34,12 @@ async function syncIndex(version, handler) { const sourceFilename = version > 1 ? `source${version}.msix` : 'source.msix'; try { // download index package to buffer - const [indexBuffer, modifiedDate] = await syncFile(sourceFilename, true, false); - if (!indexBuffer) { + const [indexBuffer, modifiedDate, updated] = await syncFile(sourceFilename, true, false); + if (!updated && !forceSync) { winston.info(`skip syncing version ${version} from ${remote}`); return; } + assert(Buffer.isBuffer(indexBuffer), 'Failed to get the source index buffer!'); // unpack, extract and load index database try { @@ -77,10 +78,10 @@ await syncIndex(2, async (db) => { try { // sync latest package metadata and manifests in parallel await async.eachLimit(packageURIs, parallelLimit, async (uri) => { - const [metadataBuffer, modifiedDate] = await syncFile(uri, false, false); + const [metadataBuffer, modifiedDate] = await syncFile(uri, forceSync, false); if (metadataBuffer) { const manifestURIs = await buildManifestURIsFromPackageMetadata(metadataBuffer); - await async.eachSeries(manifestURIs, async (uri) => await syncFile(uri, false)); + await async.eachSeries(manifestURIs, async (uri) => await syncFile(uri, forceSync)); await cacheFileWithURI(uri, metadataBuffer, modifiedDate); } }); @@ -98,7 +99,7 @@ await syncIndex(1, async (db) => { const uris = buildManifestURIs(await db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC'), pathparts); // sync latest manifests in parallel try { - await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, false)); + await async.eachLimit(uris, parallelLimit, async (uri) => await syncFile(uri, forceSync)); } catch (error) { exitWithCode(EX_TEMPFAIL, error); } diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 4ddb67b..f7034fa 100755 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -6,6 +6,7 @@ ## SET IN ENVIRONMENT VARIABLES #BIND_ADDRESS= #DEBUG= +#WINGET_FORCE_SYNC= #WINGET_REPO_URL= #WINGET_REPO_JOBS= diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 9d66a3e..df88cfb 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -49,6 +49,9 @@ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); /** Whether the debug mode is enabled. */ const debugMode = process.env.DEBUG === 'true'; +/** Whether to perform a forced sync. */ +const forceSync = process.env.WINGET_FORCE_SYNC === 'true'; + /** Local IP address to be bound to HTTPS requests. */ const localAddress = process.env.BIND_ADDRESS; @@ -305,6 +308,7 @@ export function setupEnvironment() { } return { debugMode, + forceSync, local, parallelLimit, remote, @@ -355,7 +359,7 @@ export async function syncFile(uri, update = true, save = true) { const localFile = await stat(localPath); if (localFile.mtime.getTime() == lastModified.getTime() && localFile.size == contentLength) { winston.debug(`skipped ${uri} because it's up to date`); - return [null, lastModified, false]; + return [await readFile(localPath), lastModified, false]; } } } From 6789ea87b79370fbee26d49feb95d123132fe841 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 25 Jul 2024 12:58:02 +0800 Subject: [PATCH 4/5] winget-source: bump version --- winget-source/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winget-source/package.json b/winget-source/package.json index 5017925..a25d438 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -1,6 +1,6 @@ { "name": "@ustcmirror/winget-source", - "version": "1.2.0", + "version": "1.2.1", "description": "Sync with pre-indexed WinGet source repository.", "main": "sync-repo.js", "author": "YR Chen ", From cb04502cf4a9fdf30909a78003a797e195347166 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 25 Jul 2024 13:49:51 +0800 Subject: [PATCH 5/5] winget-source: recude disk write on force sync --- winget-source/sync-repo.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 3185ccf..60335d5 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -58,7 +58,9 @@ async function syncIndex(version, handler) { } // update index package - await cacheFileWithURI(sourceFilename, indexBuffer, modifiedDate); + if (updated) { + await cacheFileWithURI(sourceFilename, indexBuffer, modifiedDate); + } } catch (error) { try { await rm(tempDirectory, { recursive: true }); @@ -78,11 +80,13 @@ await syncIndex(2, async (db) => { try { // sync latest package metadata and manifests in parallel await async.eachLimit(packageURIs, parallelLimit, async (uri) => { - const [metadataBuffer, modifiedDate] = await syncFile(uri, forceSync, false); + const [metadataBuffer, modifiedDate, updated] = await syncFile(uri, forceSync, false); if (metadataBuffer) { const manifestURIs = await buildManifestURIsFromPackageMetadata(metadataBuffer); await async.eachSeries(manifestURIs, async (uri) => await syncFile(uri, forceSync)); - await cacheFileWithURI(uri, metadataBuffer, modifiedDate); + if (updated) { + await cacheFileWithURI(uri, metadataBuffer, modifiedDate); + } } }); } catch (error) {