diff --git a/package.json b/package.json index a04bbb4..88567ec 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ }, "dependencies": { "@google-cloud/datastore": "^8.0.0", - "@naturalcycles/db-lib": "^8.46.1", + "@naturalcycles/db-lib": "^9.0.0", "@naturalcycles/js-lib": "^14.116.0", "@naturalcycles/nodejs-lib": "^13.1.0" }, diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 0e6b346..2ac882d 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -6,7 +6,7 @@ "compilerOptions": { "skipLibCheck": true, // because of https://github.com/dcodeIO/long.js/issues/109 "baseUrl": "./", - "outDir": "../dist" + "outDir": "../dist", }, - "exclude": ["**/__exclude"] + "exclude": ["**/__exclude"], } diff --git a/src/datastore.db.ts b/src/datastore.db.ts index dfcb6fe..24007af 100644 --- a/src/datastore.db.ts +++ b/src/datastore.db.ts @@ -1,13 +1,14 @@ import { Transform } from 'node:stream' -import { PropertyFilter } from '@google-cloud/datastore' +import { PropertyFilter, Transaction } from '@google-cloud/datastore' import type { Datastore, Key, Query } from '@google-cloud/datastore' import { BaseCommonDB, CommonDB, + commonDBFullSupport, CommonDBSaveMethod, + CommonDBSupport, DBQuery, DBTransaction, - mergeDBOperations, RunQueryResult, } from '@naturalcycles/db-lib' import { @@ -72,6 +73,11 @@ const methodMap: Record = { * https://cloud.google.com/datastore/docs/datastore-api-tutorial */ export class DatastoreDB extends BaseCommonDB implements CommonDB { + override support: CommonDBSupport = { + ...commonDBFullSupport, + updateByQuery: false, + } + constructor(cfg: DatastoreDBCfg = {}) { super() this.cfg = { @@ -125,7 +131,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB { override async getByIds( table: string, - ids: ROW['id'][], + ids: string[], _opt?: DatastoreDBOptions, ): Promise { if (!ids.length) return [] @@ -178,10 +184,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB { ) } - getQueryKind(q: Query): string { - if (!q?.kinds?.length) return '' // should never be the case, but - return q.kinds[0]! - } + // getQueryKind(q: Query): string { + // if (!q?.kinds?.length) return '' // should never be the case, but + // return q.kinds[0]! + // } override async runQuery( dbQuery: DBQuery, @@ -284,7 +290,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB { const save = pRetryFn( async (batch: DatastorePayload[]) => { - await (opt.tx || this.ds())[method](batch) + await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds())[method](batch) }, this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`), ) @@ -331,49 +337,23 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB { * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted" * regardless if they were actually deleted or not. */ - async deleteByIds( + override async deleteByIds( table: string, - ids: ROW['id'][], + ids: string[], opt: DatastoreDBOptions = {}, ): Promise { const keys = ids.map(id => this.key(table, id)) - // eslint-disable-next-line @typescript-eslint/return-await - await pMap(_chunk(keys, MAX_ITEMS), async batch => await (opt.tx || this.ds()).delete(batch)) + + await pMap( + _chunk(keys, MAX_ITEMS), + // eslint-disable-next-line @typescript-eslint/return-await + async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch), + ) return ids.length } - /** - * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs - */ - override async commitTransaction( - _tx: DBTransaction, - opt?: DatastoreDBSaveOptions, - ): Promise { - // Using Retry, because Datastore can throw errors like "too much contention" here - await pRetry(async () => { - const tx = this.ds().transaction() - - try { - await tx.run() - - const ops = mergeDBOperations(_tx.ops) - - for await (const op of ops) { - if (op.type === 'saveBatch') { - await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx }) - } else if (op.type === 'deleteByIds') { - await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx }) - } else { - throw new Error(`DBOperation not supported: ${(op as any).type}`) - } - } - - await tx.commit() - } catch (err) { - await tx.rollback() - throw err // rethrow - } - }, this.getPRetryOptions(`DatastoreLib.commitTransaction`)) + override async createTransaction(): Promise { + return await DatastoreDBTransaction.create(this) } async getAllStats(): Promise { @@ -547,3 +527,26 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB { } } } + +/** + * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs + */ +export class DatastoreDBTransaction implements DBTransaction { + private constructor( + public db: DatastoreDB, + public tx: Transaction, + ) {} + + static async create(db: DatastoreDB): Promise { + const tx = db.ds().transaction() + await tx.run() + return new DatastoreDBTransaction(db, tx) + } + + async commit(): Promise { + await this.tx.commit() + } + async rollback(): Promise { + await this.tx.rollback() + } +} diff --git a/src/datastore.model.ts b/src/datastore.model.ts index 064747d..ea7b35b 100644 --- a/src/datastore.model.ts +++ b/src/datastore.model.ts @@ -1,4 +1,4 @@ -import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore' +import type { DatastoreOptions, Key } from '@google-cloud/datastore' import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib' import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib' @@ -114,13 +114,10 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions { maxWait?: NumberOfSeconds } -export interface DatastoreDBOptions extends CommonDBOptions { - tx?: Transaction -} +export interface DatastoreDBOptions extends CommonDBOptions {} + export interface DatastoreDBSaveOptions = any> - extends CommonDBSaveOptions { - tx?: Transaction -} + extends CommonDBSaveOptions {} export interface DatastoreStats { composite_index_count: number diff --git a/src/index.ts b/src/index.ts index dea278c..0259819 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,27 +1,3 @@ -import { DatastoreDB } from './datastore.db' -import { - DatastoreCredentials, - DatastoreDBCfg, - DatastoreDBOptions, - DatastoreDBSaveOptions, - DatastoreDBStreamOptions, - DatastorePayload, - DatastorePropertyStats, - DatastoreStats, - DatastoreType, -} from './datastore.model' -import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB' - -export type { - DatastorePayload, - DatastoreDBCfg, - DatastoreKeyValueDBCfg, - DatastoreStats, - DatastoreCredentials, - DatastorePropertyStats, - DatastoreDBStreamOptions, - DatastoreDBOptions, - DatastoreDBSaveOptions, -} - -export { DatastoreDB, DatastoreType, DatastoreKeyValueDB } +export * from './datastore.db' +export * from './datastore.model' +export * from './datastoreKeyValueDB' diff --git a/src/test/datastore.manual.test.ts b/src/test/datastore.manual.test.ts index 43e04fa..c242096 100644 --- a/src/test/datastore.manual.test.ts +++ b/src/test/datastore.manual.test.ts @@ -1,6 +1,4 @@ import { - CommonDBImplementationFeatures, - CommonDBImplementationQuirks, createTestItemsDBM, runCommonDaoTest, runCommonDBTest, @@ -26,20 +24,9 @@ export const datastoreDB = new DatastoreDB({ }, }) -// Seems like consistency quirks are no longer needed? -// UPD 2021-08-05: nope, still needed -const features: CommonDBImplementationFeatures = { - // strongConsistency: false, - updateByQuery: false, -} -const quirks: CommonDBImplementationQuirks = { - // 2021-10-07: fails when set to 100, bumped up to 300 - // eventualConsistencyDelay: 300, -} +describe('runCommonDBTest', () => runCommonDBTest(datastoreDB)) -describe('runCommonDBTest', () => runCommonDBTest(datastoreDB, features, quirks)) - -describe('runCommonDaoTest', () => runCommonDaoTest(datastoreDB, features, quirks)) +describe('runCommonDaoTest', () => runCommonDaoTest(datastoreDB)) test('getStats, getStatsCount non-existing table', async () => { expect(await datastoreDB.getStats('NonEx')).toBeUndefined() diff --git a/tsconfig.json b/tsconfig.json index fd7faac..e353a14 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,8 +2,8 @@ "extends": "@naturalcycles/dev-lib/cfg/tsconfig.json", "compilerOptions": { "skipLibCheck": true, // because of https://github.com/dcodeIO/long.js/issues/109 - "outDir": "./dist" + "outDir": "./dist", }, "include": ["src"], - "exclude": ["**/__exclude"] + "exclude": ["**/__exclude"], } diff --git a/yarn.lock b/yarn.lock index 4c6e8e4..3f93a84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -527,9 +527,9 @@ integrity sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g== "@grpc/grpc-js@~1.9.6": - version "1.9.13" - resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.9.13.tgz#ad9b7dbb6089c462469653c809996f13e46aa1cd" - integrity sha512-OEZZu9v9AA+7/tghMDE8o5DAMD5THVnwSqDWuh7PPYO5287rTyqy0xEHT6/e4pbqSrhyLPdQFsam4TwFQVVIIw== + version "1.9.14" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.9.14.tgz#236378822876cbf7903f9d61a0330410e8dcc5a1" + integrity sha512-nOpuzZ2G3IuMFN+UPPpKrC6NsLmWsTqSsm66IRfnBt1D4pwTqE27lmbpcPM+l2Ua4gE7PfjRHI6uedAy7hoXUw== dependencies: "@grpc/proto-loader" "^0.7.8" "@types/node" ">=12.12.47" @@ -835,10 +835,10 @@ tslib "^2.6.2" typescript "^5.0.2" -"@naturalcycles/db-lib@^8.46.1": - version "8.60.0" - resolved "https://registry.yarnpkg.com/@naturalcycles/db-lib/-/db-lib-8.60.0.tgz#320af2c5fb0a65f7b545cd949b6e24e42eb51d9e" - integrity sha512-57zb4y9nC8itKp5VkcCHxESTU6Oil3vQJ6rqmlfdSEL4UXrN6Nu/+wt7NuP4ZDQq57OmFpXbA1vnG6scCCwdoA== +"@naturalcycles/db-lib@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@naturalcycles/db-lib/-/db-lib-9.0.0.tgz#5f3a3e80d1de02d24e1f17213265700ab0a4dbfe" + integrity sha512-+d8AIXPOt9FgxgagNyweilde1CJp2TwK2yf0vw98x4JMgYmycoF0MgZjXvyDICh0hzOn3vj3Arxve0jLFrbduQ== dependencies: "@naturalcycles/js-lib" "^14.116.0" "@naturalcycles/nodejs-lib" "^13.1.1" @@ -887,9 +887,9 @@ zod "^3.20.2" "@naturalcycles/nodejs-lib@^13.0.1", "@naturalcycles/nodejs-lib@^13.1.0", "@naturalcycles/nodejs-lib@^13.1.1": - version "13.6.0" - resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.6.0.tgz#33acd08d9104499ad37ffdafbac208f774ee4493" - integrity sha512-9Ol32xy24dmP38qhWCfZM8QkgLCsRRki+GFZjJeJQh0q42WhACahAwac9TF8XWCAEXQkOm1ft8zw/PQknEVNfw== + version "13.7.0" + resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.7.0.tgz#8d49d85d9c3165f20a9bbc08024bc213f5318cae" + integrity sha512-mPGMLS5pBP8U9ToVwM91+7ydir+fCUqYE3NVDotvfQW6jYf+LExKqnyMDGCEnwkGZhkafvGHbHBh5eqluyXsWA== dependencies: "@naturalcycles/js-lib" "^14.0.0" "@types/js-yaml" "^4.0.9" @@ -1151,9 +1151,9 @@ integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag== "@types/node@*", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@^20.1.0", "@types/node@^20.4.1": - version "20.11.4" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.4.tgz#c724a5d6723182af758b91b994209336f4439cb7" - integrity sha512-6I0fMH8Aoy2lOejL3s4LhyIYX34DPwY8bl5xlNjBvUEk8OHrcuzsFt+Ied4LvJihbtXPM+8zUqdydfIti86v9g== + version "20.11.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.5.tgz#be10c622ca7fcaa3cf226cf80166abc31389d86e" + integrity sha512-g557vgQjUUfN76MZAN/dt1z3dzcUsimuysco0KeluHgrPdJXkP/XdAURgyO2W9fZWHRtRBiVKzKn8vyOAwlG+w== dependencies: undici-types "~5.26.4" @@ -1791,9 +1791,9 @@ camelcase@^6.2.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001565: - version "1.0.30001577" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001577.tgz#a24991eb4ad67324ba8b96716340d53151f2f6f8" - integrity sha512-rs2ZygrG1PNXMfmncM0B5H1hndY5ZCC9b5TkFaVNfZ+AUlyqcMyVIQtc3fsezi0NUCk5XZfDf9WS6WxMxnfdrg== + version "1.0.30001578" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001578.tgz#11741580434ce60aae4b4a9abee9f9f8d7bf5be5" + integrity sha512-J/jkFgsQ3NEl4w2lCoM9ZPxrD+FoBNJ7uJUpGVjIg/j0OwJosWM36EPDv+Yyi0V4twBk9pPmlFS+PLykgEvUmg== chalk@5.3.0: version "5.3.0" @@ -2173,9 +2173,9 @@ ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: safe-buffer "^5.0.1" electron-to-chromium@^1.4.601: - version "1.4.632" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.632.tgz#df6253483b802eb83eee2fdc0e5067bd46f36f11" - integrity sha512-JGmudTwg7yxMYvR/gWbalqqQiyu7WTFv2Xu3vw4cJHXPFxNgAk0oy8UHaer8nLF4lZJa+rNoj6GsrKIVJTV6Tw== + version "1.4.635" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.635.tgz#e4e064b8711a98827652ce17cc11b0e0184c40d1" + integrity sha512-iu/2D0zolKU3iDGXXxdOzNf72Jnokn+K1IN6Kk4iV6l1Tr2g/qy+mvmtfAiBwZe5S3aB5r92vp+zSZ69scYRrg== emittery@^0.13.1: version "0.13.1" @@ -4490,9 +4490,9 @@ prelude-ls@^1.2.1: integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== prettier@^3.0.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.2.tgz#96e580f7ca9c96090ad054616c0c4597e2844b65" - integrity sha512-HTByuKZzw7utPiDO523Tt2pLtEyK7OibUD9suEJQrPUCYQqrHr74GGX6VidMrovbf/I50mPqr8j/II6oBAuc5A== + version "3.2.4" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.4.tgz#4723cadeac2ce7c9227de758e5ff9b14e075f283" + integrity sha512-FWu1oLHKCrtpO1ypU6J0SbK2d9Ckwysq6bHj/uaCP26DxrPpppCLQRGVuqAxSTvhF00AcvDRyYrLNW7ocBhFFQ== pretty-format@^29.0.0, pretty-format@^29.7.0: version "29.7.0" @@ -4523,7 +4523,7 @@ proto3-json-serializer@^2.0.0: dependencies: protobufjs "^7.2.5" -protobufjs@7.2.5, protobufjs@^7.2.4, protobufjs@^7.2.5: +protobufjs@7.2.5: version "7.2.5" resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.5.tgz#45d5c57387a6d29a17aab6846dcc283f9b8e7f2d" integrity sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A== @@ -4541,6 +4541,24 @@ protobufjs@7.2.5, protobufjs@^7.2.4, protobufjs@^7.2.5: "@types/node" ">=13.7.0" long "^5.0.0" +protobufjs@^7.2.4, protobufjs@^7.2.5: + version "7.2.6" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.6.tgz#4a0ccd79eb292717aacf07530a07e0ed20278215" + integrity sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"