From cc1ac80c37abb78a877343e3b2e6d4e37d8c58bc Mon Sep 17 00:00:00 2001 From: jorgeer Date: Thu, 13 Oct 2022 20:21:31 +0200 Subject: [PATCH 01/50] Add redis-oplog compatibility for live updates --- lib/hijack/instrument.js | 6 +- lib/hijack/wrap_observers.js | 132 ++++++++++++++++++++++++++++++++--- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/lib/hijack/instrument.js b/lib/hijack/instrument.js index e138494a..b104fbb1 100644 --- a/lib/hijack/instrument.js +++ b/lib/hijack/instrument.js @@ -28,7 +28,11 @@ Kadira._startInstrumenting = function (callback) { wrapSubscription(MeteorX.Subscription.prototype); if (MeteorX.MongoOplogDriver) { - wrapOplogObserveDriver(MeteorX.MongoOplogDriver.prototype); + if (MeteorX.MongoOplogDriver.name === "RedisOplogObserveDriver") { + wrapRedisOplogObserveDriver(MeteorX.MongoOplogDriver); + } else { + wrapOplogObserveDriver(MeteorX.MongoOplogDriver.prototype); + } } if (MeteorX.MongoPollingDriver) { diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 7155b048..2945ba74 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -95,6 +95,115 @@ wrapOplogObserveDriver = function (proto) { }; }; +wrapRedisOplogObserveDriver = function (driver) { + const driverProto = driver.prototype; + const MongoColl = typeof Mongo !== "undefined" ? Mongo.Collection : Meteor.Collection; + const collName = "__dummy_coll_" + Random.id(); + const coll = new MongoColl(collName); + const dummyDriver = new driver({ + cursorDescription: { + collectionName: collName, + selector: {}, + options: {} + }, + multiplexer: { ready () {} }, + matcher: { combineIntoProjection: () => ({}) }, + }); + + const observableCollectionProto = dummyDriver.observableCollection.constructor.prototype; + const redisSubscriberProto = dummyDriver.redisSubscriber.constructor.prototype; + + let originalAdd = observableCollectionProto.add; + let originalChange = observableCollectionProto.change; + let originalRemove = observableCollectionProto.remove; + // Track the polled documents. This is reflect to the RAM size and + // for the CPU usage directly + + observableCollectionProto.add = function (doc, safe) { + let coll = this.cursorDescription.collectionName; + let query = this.cursorDescription.selector; + let opts = this.cursorDescription.options; + var docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); + + // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. + // But it is used in the initial add and syntetic mutations, so we use it to get initial adds. + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_addPublished", 1); + Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); + Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'polledFetches', docSize); + } else { + if (safe) { + this._polledDocuments = 1; + this._docSize = { + polledFetches: docSize + }; + } else { + // If there is no ownerInfo, that means this is the initial adds + if (!this._liveUpdatesCounts) { + this._liveUpdatesCounts = { + _initialAdds: 0 + }; + } + + this._liveUpdatesCounts._initialAdds++; + + if (!this._docSize) { + this._docSize = { + initialFetches: 0 + }; + } + + if (!this._docSize.initialFetches) { + this._docSize.initialFetches = 0; + } + + this._docSize.initialFetches += docSize; + } + } + + return originalAdd.call(this, doc, safe); + } + + observableCollectionProto.change = function (doc, modifiedFields) { + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_changePublished", 1); + } + originalChange.call(this, doc, modifiedFields); + } + + observableCollectionProto.remove = function (docId) { + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_removePublished", 1); + } + originalRemove.call(this, docId); + } + + // Redis and oplog don't have the same op constants + const redisEventToOplogMap = { + i: "i", + u: "u", + r: "d", + } + + const originalRedisProcess = redisSubscriberProto.process; + redisSubscriberProto.process = function (op, doc, fields, ...rest) { + Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { + op: redisEventToOplogMap[op] + }); + originalRedisProcess.call(this, op, doc, fields); + } + + let originalStop = driverProto.stop; + driverProto.stop = function () { + if (this.observableCollection._ownerInfo && this.observableCollection._ownerInfo.type === 'sub') { + Kadira.EventBus.emit('pubsub', 'observerDeleted', this.observableCollection._ownerInfo); + Kadira.models.pubsub.trackDeletedObserver(this.observableCollection._ownerInfo); + } + + return originalStop.call(this); + }; +}; + wrapPollingObserveDriver = function (proto) { let originalPollMongo = proto._pollMongo; proto._pollMongo = function () { @@ -160,6 +269,7 @@ wrapForCountingObservers = function () { // to count observers let mongoConnectionProto = MeteorX.MongoConnection.prototype; let originalObserveChanges = mongoConnectionProto._observeChanges; + mongoConnectionProto._observeChanges = function (cursorDescription, ordered, callbacks) { let ret = originalObserveChanges.call(this, cursorDescription, ordered, callbacks); // get the Kadira Info via the Meteor.EnvironmentalVariable @@ -179,29 +289,33 @@ wrapForCountingObservers = function () { }; let observerDriver = ret._multiplexer._observeDriver; - observerDriver._ownerInfo = ownerInfo; + // We store counts for redis-oplog in the observableCollection instead + let ownerStorer = observerDriver.observableCollection || observerDriver; + + ownerStorer._ownerInfo = ownerInfo; + Kadira.EventBus.emit('pubsub', 'observerCreated', ownerInfo); Kadira.models.pubsub.trackCreatedObserver(ownerInfo); // We need to send initially polled documents if there are - if (observerDriver._polledDocuments) { - Kadira.models.pubsub.trackPolledDocuments(ownerInfo, observerDriver._polledDocuments); - observerDriver._polledDocuments = 0; + if (ownerStorer._polledDocuments) { + Kadira.models.pubsub.trackPolledDocuments(ownerInfo, ownerStorer._polledDocuments); + ownerStorer._polledDocuments = 0; } // We need to send initially polled documents if there are - if (observerDriver._polledDocSize) { - Kadira.models.pubsub.trackDocSize(ownerInfo.name, 'polledFetches', observerDriver._polledDocSize); - observerDriver._polledDocSize = 0; + if (ownerStorer._polledDocSize) { + Kadira.models.pubsub.trackDocSize(ownerInfo.name, 'polledFetches', ownerStorer._polledDocSize); + ownerStorer._polledDocSize = 0; } // Process _liveUpdatesCounts - _.each(observerDriver._liveUpdatesCounts, function (count, key) { + _.each(ownerStorer._liveUpdatesCounts, function (count, key) { Kadira.models.pubsub.trackLiveUpdates(ownerInfo, key, count); }); // Process docSize - _.each(observerDriver._docSize, function (count, key) { + _.each(ownerStorer._docSize, function (count, key) { Kadira.models.pubsub.trackDocSize(ownerInfo.name, key, count); }); } From 2924c0625fc3c7c69277424fbaaaa4ffff40f415 Mon Sep 17 00:00:00 2001 From: jorgeer Date: Mon, 17 Oct 2022 13:27:12 +0200 Subject: [PATCH 02/50] fix document counting redis-oplog --- lib/hijack/wrap_observers.js | 43 +++++++++++++++++------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 2945ba74..565d6ac6 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -124,7 +124,6 @@ wrapRedisOplogObserveDriver = function (driver) { let query = this.cursorDescription.selector; let opts = this.cursorDescription.options; var docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); - // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. // But it is used in the initial add and syntetic mutations, so we use it to get initial adds. if (this._ownerInfo) { @@ -132,33 +131,31 @@ wrapRedisOplogObserveDriver = function (driver) { Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'polledFetches', docSize); } else { - if (safe) { - this._polledDocuments = 1; - this._docSize = { - polledFetches: docSize + // If there is no ownerInfo, that means this is the initial adds + if (!this._liveUpdatesCounts) { + this._liveUpdatesCounts = { + _initialAdds: 0 }; - } else { - // If there is no ownerInfo, that means this is the initial adds - if (!this._liveUpdatesCounts) { - this._liveUpdatesCounts = { - _initialAdds: 0 - }; - } - - this._liveUpdatesCounts._initialAdds++; + } - if (!this._docSize) { - this._docSize = { - initialFetches: 0 - }; - } + this._liveUpdatesCounts._initialAdds++; - if (!this._docSize.initialFetches) { - this._docSize.initialFetches = 0; - } + if (this._polledDocuments) { + this._polledDocuments += 1; + } else { + this._polledDocuments = 1; + } - this._docSize.initialFetches += docSize; + if (this._docSize) { + this._docSize.polledFetches += docSize; + } else { + this._docSize = { + polledFetches: docSize, + initialFetches: 0 + }; } + + this._docSize.initialFetches += docSize; } return originalAdd.call(this, doc, safe); From 41b536cd44e07f893420a4e35e29daaf69cf9aac Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 27 Sep 2023 09:08:16 -0400 Subject: [PATCH 03/50] adjust .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 697870b5..8b7dbbe1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ smart.lock versions.json .idea .eslintcache +.envrc \ No newline at end of file From 43c5b3a43c27ab7f3abff41c8d1e38e5caa343cc Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 27 Sep 2023 09:14:01 -0400 Subject: [PATCH 04/50] fix lint issues --- lib/hijack/instrument.js | 5 +++-- lib/hijack/wrap_observers.js | 38 ++++++++++++++++++------------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/hijack/instrument.js b/lib/hijack/instrument.js index b69cf11e..ac54afa6 100644 --- a/lib/hijack/instrument.js +++ b/lib/hijack/instrument.js @@ -11,7 +11,8 @@ import { wrapForCountingObservers, wrapMultiplexer, wrapOplogObserveDriver, - wrapPollingObserveDriver + wrapPollingObserveDriver, + wrapRedisOplogObserveDriver } from './wrap_observers'; import { wrapStringifyDDP } from './wrap_ddp_stringify'; import { setLabels } from './set_labels'; @@ -40,7 +41,7 @@ Kadira._startInstrumenting = function (callback) { wrapSubscription(MeteorX.Subscription.prototype); if (MeteorX.MongoOplogDriver) { - if (MeteorX.MongoOplogDriver.name === "RedisOplogObserveDriver") { + if (MeteorX.MongoOplogDriver.name === 'RedisOplogObserveDriver') { wrapRedisOplogObserveDriver(MeteorX.MongoOplogDriver); } else { wrapOplogObserveDriver(MeteorX.MongoOplogDriver.prototype); diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 5afd3c33..995430cc 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -1,4 +1,5 @@ import { _ } from 'meteor/underscore'; +import { Random } from 'meteor/random'; export function wrapOplogObserveDriver (proto) { // Track the polled documents. This is reflected to the RAM size and @@ -94,11 +95,10 @@ export function wrapOplogObserveDriver (proto) { }; } -export function wrapRedisOplogObserveDriver(driver) { +export function wrapRedisOplogObserveDriver (driver) { const driverProto = driver.prototype; - const MongoColl = typeof Mongo !== "undefined" ? Mongo.Collection : Meteor.Collection; - const collName = "__dummy_coll_" + Random.id(); - const coll = new MongoColl(collName); + const collName = `__dummy_coll_${Random.id()}`; + const dummyDriver = new driver({ cursorDescription: { collectionName: collName, @@ -122,11 +122,11 @@ export function wrapRedisOplogObserveDriver(driver) { let coll = this.cursorDescription.collectionName; let query = this.cursorDescription.selector; let opts = this.cursorDescription.options; - var docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); + let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. // But it is used in the initial add and syntetic mutations, so we use it to get initial adds. if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_addPublished", 1); + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'polledFetches', docSize); } else { @@ -158,36 +158,36 @@ export function wrapRedisOplogObserveDriver(driver) { } return originalAdd.call(this, doc, safe); - } + }; observableCollectionProto.change = function (doc, modifiedFields) { if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_changePublished", 1); + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_changePublished', 1); } originalChange.call(this, doc, modifiedFields); - } + }; observableCollectionProto.remove = function (docId) { if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, "_removePublished", 1); + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_removePublished', 1); } originalRemove.call(this, docId); - } + }; // Redis and oplog don't have the same op constants const redisEventToOplogMap = { - i: "i", - u: "u", - r: "d", - } + i: 'i', + u: 'u', + r: 'd', + }; const originalRedisProcess = redisSubscriberProto.process; - redisSubscriberProto.process = function (op, doc, fields, ...rest) { + redisSubscriberProto.process = function (op, doc, fields) { Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { op: redisEventToOplogMap[op] }); originalRedisProcess.call(this, op, doc, fields); - } + }; let originalStop = driverProto.stop; driverProto.stop = function () { @@ -198,9 +198,9 @@ export function wrapRedisOplogObserveDriver(driver) { return originalStop.call(this); }; -}; +} -export function wrapPollingObserveDriver(proto) { +export function wrapPollingObserveDriver (proto) { let originalPollMongo = proto._pollMongo; proto._pollMongo = function () { originalPollMongo.call(this); From 400bca2c662a040164377d50eb533d1bdac1499c Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 27 Sep 2023 14:09:15 -0400 Subject: [PATCH 05/50] fix non existent collection error --- lib/hijack/wrap_observers.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 995430cc..93e25f0d 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -1,5 +1,7 @@ import { _ } from 'meteor/underscore'; import { Random } from 'meteor/random'; +import { Mongo } from 'meteor/mongo'; +import { Meteor } from 'meteor/meteor'; export function wrapOplogObserveDriver (proto) { // Track the polled documents. This is reflected to the RAM size and @@ -97,11 +99,12 @@ export function wrapOplogObserveDriver (proto) { export function wrapRedisOplogObserveDriver (driver) { const driverProto = driver.prototype; - const collName = `__dummy_coll_${Random.id()}`; + const MongoColl = typeof Mongo !== 'undefined' ? Mongo.Collection : Meteor.Collection; + const collection = new MongoColl(`__dummy_coll_${Random.id()}`); const dummyDriver = new driver({ cursorDescription: { - collectionName: collName, + collectionName: collection._name, selector: {}, options: {} }, From 39c7cddd449a33ddf915faaf3804f99188135972 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Thu, 28 Sep 2023 09:34:52 -0400 Subject: [PATCH 06/50] add test helpers --- package-lock.json | 232 ++++++++++++++++++++++++++++++++--- package.json | 6 +- tests/_helpers/pretty-log.js | 53 ++++++++ 3 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 tests/_helpers/pretty-log.js diff --git a/package-lock.json b/package-lock.json index 9049377f..e8a6da1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -599,6 +599,12 @@ "color-convert": "^2.0.1" } }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -608,7 +614,7 @@ "array-includes": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz", - "integrity": "sha512-mRVEsI0s5MycUKtZtn8i5co54WKxL5gH3gAcCjtUbECNwdDL2gsBwjLqswM3c6fjcuWFQ9hoS4C+EhjxQmEyHQ==", + "integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=", "dev": true, "requires": { "define-properties": "^1.1.2", @@ -706,6 +712,43 @@ "supports-color": "^7.1.0" } }, + "cli-highlight": { + "version": "2.1.11", + "resolved": "https://registry.npmjs.org/cli-highlight/-/cli-highlight-2.1.11.tgz", + "integrity": "sha512-9KDcoEVwyUXrjcJNvHD0NFc/hiwe/WPVYIleQh2O1N2Zro5gWJZ/K+3DGn8w8P/F6FxOgzyC5bxDyHIgCSPhGg==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "highlight.js": "^10.7.1", + "mz": "^2.4.0", + "parse5": "^5.1.1", + "parse5-htmlparser2-tree-adapter": "^6.0.0", + "yargs": "^16.0.0" + }, + "dependencies": { + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + } + } + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -724,7 +767,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, "convert-source-map": { @@ -775,6 +818,12 @@ "object-keys": "^1.1.1" } }, + "diff": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", + "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "dev": true + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -790,6 +839,12 @@ "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, "es-abstract": { "version": "1.21.0", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.21.0.tgz", @@ -908,6 +963,18 @@ "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", "text-table": "^0.2.0" + }, + "dependencies": { + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + } } }, "eslint-config-es": { @@ -1023,7 +1090,7 @@ "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, "fastq": { @@ -1082,7 +1149,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, "function-bind": { @@ -1115,6 +1182,12 @@ "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", "dev": true }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, "get-intrinsic": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", @@ -1243,6 +1316,12 @@ "has-symbols": "^1.0.2" } }, + "highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", + "dev": true + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -1268,13 +1347,13 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "requires": { "once": "^1.3.0", @@ -1345,7 +1424,13 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, "is-glob": { @@ -1440,7 +1525,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, "js-sdsl": { @@ -1479,7 +1564,7 @@ "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, "json5": { @@ -1528,10 +1613,21 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "requires": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, "node-releases": { @@ -1540,6 +1636,12 @@ "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", "dev": true }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true + }, "object-inspect": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", @@ -1567,7 +1669,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "requires": { "wrappy": "1" @@ -1614,6 +1716,29 @@ "callsites": "^3.0.0" } }, + "parse5": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", + "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==", + "dev": true + }, + "parse5-htmlparser2-tree-adapter": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz", + "integrity": "sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==", + "dev": true, + "requires": { + "parse5": "^6.0.1" + }, + "dependencies": { + "parse5": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-6.0.1.tgz", + "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", + "dev": true + } + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -1623,7 +1748,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, "path-key": { @@ -1673,6 +1798,12 @@ "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", "dev": true }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, "resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -1740,6 +1871,17 @@ "object-inspect": "^1.9.0" } }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "string.prototype.trimend": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.6.tgz", @@ -1789,9 +1931,27 @@ "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "requires": { + "any-promise": "^1.0.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "requires": { + "thenify": ">= 3.1.0 < 4" + } + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -1897,10 +2057,54 @@ "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", "dev": true }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + }, + "yaml": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz", + "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==", + "dev": true + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } + }, + "yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true }, "yocto-queue": { diff --git a/package.json b/package.json index 5e18c71c..3aec70b1 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,13 @@ "@babel/core": "^7.20.2", "@babel/eslint-parser": "^7.16.5", "@types/meteor": "^1.4.87", + "chalk": "^4.1.2", + "cli-highlight": "^2.1.11", + "diff": "^5.1.0", "eslint": "^8.31.0", "eslint-config-es": "^0.8.12", - "eslint-plugin-babel": "^5.3.1" + "eslint-plugin-babel": "^5.3.1", + "yaml": "^2.3.2" }, "dependencies": {}, "volta": { diff --git a/tests/_helpers/pretty-log.js b/tests/_helpers/pretty-log.js new file mode 100644 index 00000000..2099175e --- /dev/null +++ b/tests/_helpers/pretty-log.js @@ -0,0 +1,53 @@ +import chalk from 'chalk'; +import highlight from 'cli-highlight'; +import { stringify } from 'yaml'; +import Diff from 'diff'; + +export function diffStrings (a, b) { + const diff = Diff.diffLines(b, a); + + let result = ''; + + diff.forEach((part) => { + const color = part.added ? 'green' : part.removed ? 'red' : 'grey'; + result += chalk[color](part.value); + }); + + return result; +} + +export function sortedStringify (obj) { + return stringify(obj, (_, value) => { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return Object.keys(value) + .sort() + .reduce((sorted, key) => { + sorted[key] = value[key]; + return sorted; + }, {}); + } + return value; + }); +} + +export function diffObjects (a, b) { + const yamlA = sortedStringify(a); + const yamlB = sortedStringify(b); + + console.log(diffStrings(yamlA, yamlB)); +} + +export function prettyLog (...args) { + const label = args.length > 1 ? args[0] : ''; + const obj = args.length > 1 ? args[1] : args[0]; + + const yaml = stringify(obj); + + if (label) { + console.log(`${chalk.yellowBright(`${label}`)}`); + } + + console.log( + highlight(yaml?.trim() ?? '', { language: 'yaml', ignoreIllegals: true }), + ); +} From a9257066d07483aa00ee2f2e2e1e65aa85bc3604 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Fri, 29 Sep 2023 12:51:31 -0400 Subject: [PATCH 07/50] ignore find and wrap findOne --- lib/hijack/db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index e72f2fa7..87e108a2 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -51,7 +51,7 @@ export function hijackDBOps () { // upsert is handles by update // 2.4 replaced _ensureIndex with createIndex [ - 'find', 'update', 'remove', 'insert', '_ensureIndex', '_dropIndex', 'createIndex' + 'findOne', 'update', 'remove', 'insert', '_ensureIndex', '_dropIndex', 'createIndex' ].forEach(function (func) { let originalFunc = mongoConnectionProto[func]; From 29e9708d4945a4425ea5cdc5166bd6cb07374ebd Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 09:46:16 -0400 Subject: [PATCH 08/50] fix tests --- lib/hijack/db.js | 6 +++++ package.json | 2 +- tests/hijack/async.js | 3 +-- tests/hijack/db.js | 52 +++++++++++++++++-------------------------- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index 87e108a2..0eaa15e2 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -65,6 +65,12 @@ export function hijackDBOps () { func, }; + if (func === 'findOne' && mod?.projection) { + payload.projection = JSON.stringify(mod.projection); + } else if (func === 'findOne' && mod?.fields) { + payload.fields = JSON.stringify(mod.fields); + } + if (func === 'insert') { // add nothing more to the payload } else if (func === '_ensureIndex' || func === '_dropIndex' || func === 'createIndex') { diff --git a/package.json b/package.json index 3aec70b1..4cd0c766 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "version": "0.0.0", "scripts": { - "test": "meteor test-packages ./", + "test": "FORCE_COLOR=true meteor test-packages ./", "lint": "eslint . --cache . --ext .js", "publish": "npm prune --production && meteor publish" }, diff --git a/tests/hijack/async.js b/tests/hijack/async.js index bbf4976a..515c0b6a 100644 --- a/tests/hijack/async.js +++ b/tests/hijack/async.js @@ -60,7 +60,6 @@ Tinytest.add( } ); - Tinytest.add( 'Async - end event on throwInto', function (test) { @@ -74,7 +73,7 @@ Tinytest.add( }), ); } catch (err) { - TestData.find({}); + TestData.find({}).fetch(); return Kadira._getInfo(); } diff --git a/tests/hijack/db.js b/tests/hijack/db.js index 7113940a..49f341f4 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -142,15 +142,10 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: 'aa'})}], ['db',undefined,{ coll: 'tinytest-data', - func: 'fetch', - cursor: true, - selector: JSON.stringify({_id: 'aa'}), - docsFetched: 1, - docSize: JSON.stringify({_id: 'aa', dd: 10}).length, - limit: 1 + func: 'findOne', + selector: JSON.stringify({_id: 'aa'}) }], ['complete'] ]; @@ -164,6 +159,7 @@ Tinytest.add( Tinytest.add( 'Database - findOne with sort and fields', function (test) { + CleanTestData(); EnableTrackingMethods(); TestData.insert({_id: 'aa', dd: 10}); let methodId = RegisterMethod(function () { @@ -178,25 +174,22 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: 'aa'})}], ['db',undefined,{ coll: 'tinytest-data', - func: 'fetch', - cursor: true, + func: 'findOne', selector: JSON.stringify({_id: 'aa'}), - sort: JSON.stringify({dd: -1}), - docsFetched: 1, - docSize: JSON.stringify({_id: 'aa', dd: 10}).length, - limit: 1 }], ['complete'] ]; + const projection = JSON.stringify({dd: 1}); - if (events[3][2].projection) { - expected[3][2].projection = projection; + + if (events[2][2].projection) { + expected[2][2].projection = projection; } else { - expected[3][2].fields = projection; + expected[2][2].fields = projection; } + test.equal(result, {_id: 'aa', dd: 10}); test.equal(events, expected); CleanTestData(); @@ -291,7 +284,6 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'count', selector: JSON.stringify({})}], ['complete'] ]; @@ -316,7 +308,6 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: {$exists: true}})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'fetch', selector: JSON.stringify({_id: {$exists: true}}), docsFetched: 2, docSize: JSON.stringify({_id: 'aa'}).length * 2}], ['complete'] ]; @@ -343,7 +334,6 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: {$exists: true}})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'map', selector: JSON.stringify({_id: {$exists: true}}), docsFetched: 2}], ['complete'] ]; @@ -372,7 +362,6 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: {$exists: true}})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'forEach', selector: JSON.stringify({_id: {$exists: true}})}], ['complete'] ]; @@ -402,7 +391,6 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({_id: {$exists: true}})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'forEach', selector: JSON.stringify({_id: {$exists: true}})}], ['complete'] ]; @@ -416,6 +404,7 @@ Tinytest.add( 'Database - Cursor - observeChanges', function (test) { EnableTrackingMethods(); + CleanTestData(); TestData.insert({_id: 'aa'}); TestData.insert({_id: 'bb'}); let methodId = RegisterMethod(function () { @@ -432,16 +421,15 @@ Tinytest.add( let client = GetMeteorClient(); let result = client.call(methodId); let events = GetLastMethodEvents([0, 2]); - events[3][2].oplog = false; + events[2][2].oplog = false; let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'observeChanges', selector: JSON.stringify({}), oplog: false, noOfCachedDocs: 2, wasMultiplexerReady: false}], ['complete'] ]; test.equal(result, [{_id: 'aa'}, {_id: 'bb'}]); - clearAdditionalObserverInfo(events[3][2]); + clearAdditionalObserverInfo(events[2][2]); test.equal(events, expected); CleanTestData(); } @@ -450,6 +438,7 @@ Tinytest.add( Tinytest.add( 'Database - Cursor - observeChanges:re-using-multiflexer', function (test) { + CleanTestData(); EnableTrackingMethods(); TestData.insert({_id: 'aa'}); TestData.insert({_id: 'bb'}); @@ -473,20 +462,20 @@ Tinytest.add( let client = GetMeteorClient(); let result = client.call(methodId); let events = GetLastMethodEvents([0, 2]); + + events[2][2].oplog = false; events[3][2].oplog = false; - events[5][2].oplog = false; + let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'observeChanges', selector: JSON.stringify({}), oplog: false, noOfCachedDocs: 2, wasMultiplexerReady: false}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({})}], ['db',undefined,{coll: 'tinytest-data', cursor: true, func: 'observeChanges', selector: JSON.stringify({}), oplog: false, noOfCachedDocs: 2, wasMultiplexerReady: true}], ['complete'] ]; test.equal(result, [{_id: 'aa'}, {_id: 'bb'}]); + clearAdditionalObserverInfo(events[2][2]); clearAdditionalObserverInfo(events[3][2]); - clearAdditionalObserverInfo(events[5][2]); test.equal(events, expected); CleanTestData(); } @@ -511,17 +500,16 @@ Tinytest.add( let client = GetMeteorClient(); let result = client.call(methodId); let events = GetLastMethodEvents([0, 2]); - events[3][2].oplog = false; + events[2][2].oplog = false; let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'find', selector: JSON.stringify({})}], ['db',undefined,{coll: 'tinytest-data', func: 'observe', cursor: true, selector: JSON.stringify({}), oplog: false, noOfCachedDocs: 2, wasMultiplexerReady: false}], ['complete'] ]; test.equal(result, [{_id: 'aa'}, {_id: 'bb'}]); - clearAdditionalObserverInfo(events[3][2]); + clearAdditionalObserverInfo(events[2][2]); test.equal(events, expected); CleanTestData(); } From c1c5c61a4fba39df94a65e2c1085597bc38809fd Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 14:11:41 -0400 Subject: [PATCH 09/50] ignore async event --- lib/hijack/async.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/hijack/async.js b/lib/hijack/async.js index 0da09cc8..3359e9fb 100644 --- a/lib/hijack/async.js +++ b/lib/hijack/async.js @@ -1,3 +1,5 @@ +import { _ } from 'meteor/underscore'; + let Fibers = Npm.require('fibers'); let EventSymbol = Symbol('MontiEventSymbol'); let StartTracked = Symbol('MontiStartTracked'); @@ -26,8 +28,16 @@ export function wrapFibers () { let originalYield = Fibers.yield; Fibers.yield = function () { let kadiraInfo = Kadira._getInfo(); + if (kadiraInfo) { + const lastEvent = _.last(kadiraInfo.trace.events); + + if (lastEvent?.type === 'db' && lastEvent?.data?.func === 'fetch') { + return originalYield(); + } + let eventId = Kadira.tracer.event(kadiraInfo.trace, 'async'); + if (eventId) { // The event unique to this fiber // Using a symbol since Meteor doesn't copy symbols to new fibers created From ceadc45fef7b93ec4d9f45baf8a909ebd1ffbf4f Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 14:12:14 -0400 Subject: [PATCH 10/50] ignore findOne and track only fetch --- lib/hijack/db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index 0eaa15e2..890c2eeb 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -51,7 +51,7 @@ export function hijackDBOps () { // upsert is handles by update // 2.4 replaced _ensureIndex with createIndex [ - 'findOne', 'update', 'remove', 'insert', '_ensureIndex', '_dropIndex', 'createIndex' + 'update', 'remove', 'insert', '_ensureIndex', '_dropIndex', 'createIndex' ].forEach(function (func) { let originalFunc = mongoConnectionProto[func]; From 88e9250afc9ca4007c239d708c520fc708fa49a6 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 14:12:20 -0400 Subject: [PATCH 11/50] adjust tests --- tests/check_for_oplog.js | 2 +- tests/hijack/db.js | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/check_for_oplog.js b/tests/check_for_oplog.js index af428498..6a246ed6 100644 --- a/tests/check_for_oplog.js +++ b/tests/check_for_oplog.js @@ -22,7 +22,7 @@ Tinytest.addAsync('CheckForOplog - Kadira.checkWhyNoOplog - reactive publish', f const info = Kadira._getInfo(); - const event = _.last(info.trace.events); + const event = info.trace.events[2]; observeChangesEvent = _.first(event.nested); return TestData.find({}); diff --git a/tests/hijack/db.js b/tests/hijack/db.js index 49f341f4..7a02d25b 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -144,7 +144,11 @@ Tinytest.add( ['wait',undefined,{waitOn: []}], ['db',undefined,{ coll: 'tinytest-data', - func: 'findOne', + func: 'fetch', + cursor: true, + docSize: 20, + docsFetched: 1, + limit: 1, selector: JSON.stringify({_id: 'aa'}) }], ['complete'] @@ -176,8 +180,13 @@ Tinytest.add( ['wait',undefined,{waitOn: []}], ['db',undefined,{ coll: 'tinytest-data', - func: 'findOne', + func: 'fetch', + cursor: true, + docSize: 20, + docsFetched: 1, + limit: 1, selector: JSON.stringify({_id: 'aa'}), + sort: JSON.stringify({dd: -1}), }], ['complete'] ]; From ca276a965e64b6d8607db18ed5af49b4a03ce96b Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 14:22:23 -0400 Subject: [PATCH 12/50] remove unused code --- lib/hijack/db.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index 890c2eeb..202c0633 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -2,7 +2,7 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; import { Mongo, MongoInternals } from 'meteor/mongo'; import { _ } from 'meteor/underscore'; -import {haveAsyncCallback, OptimizedApply} from '../utils'; +import { haveAsyncCallback, OptimizedApply } from '../utils'; // This hijack is important to make sure, collections created before @@ -65,12 +65,6 @@ export function hijackDBOps () { func, }; - if (func === 'findOne' && mod?.projection) { - payload.projection = JSON.stringify(mod.projection); - } else if (func === 'findOne' && mod?.fields) { - payload.fields = JSON.stringify(mod.fields); - } - if (func === 'insert') { // add nothing more to the payload } else if (func === '_ensureIndex' || func === '_dropIndex' || func === 'createIndex') { From e39731a263b78c268c9fd273398c0b637ca2d86d Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 2 Oct 2023 14:23:49 -0400 Subject: [PATCH 13/50] remove null coalesce --- lib/hijack/async.js | 2 +- tests/_helpers/pretty-log.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/hijack/async.js b/lib/hijack/async.js index 3359e9fb..eec19855 100644 --- a/lib/hijack/async.js +++ b/lib/hijack/async.js @@ -32,7 +32,7 @@ export function wrapFibers () { if (kadiraInfo) { const lastEvent = _.last(kadiraInfo.trace.events); - if (lastEvent?.type === 'db' && lastEvent?.data?.func === 'fetch') { + if (lastEvent.type && lastEvent.data && lastEvent.type === 'db' && lastEvent.data.func === 'fetch') { return originalYield(); } diff --git a/tests/_helpers/pretty-log.js b/tests/_helpers/pretty-log.js index 2099175e..0f321a2d 100644 --- a/tests/_helpers/pretty-log.js +++ b/tests/_helpers/pretty-log.js @@ -41,13 +41,13 @@ export function prettyLog (...args) { const label = args.length > 1 ? args[0] : ''; const obj = args.length > 1 ? args[1] : args[0]; - const yaml = stringify(obj); + const yaml = stringify(obj) || ''; if (label) { console.log(`${chalk.yellowBright(`${label}`)}`); } console.log( - highlight(yaml?.trim() ?? '', { language: 'yaml', ignoreIllegals: true }), + highlight(yaml.trim(), { language: 'yaml', ignoreIllegals: true }), ); } From c8f39056eb8288547206312d33e899163460b99d Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Thu, 5 Oct 2023 10:26:51 -0400 Subject: [PATCH 14/50] ignore async event in tests instead --- .gitignore | 1 + lib/hijack/async.js | 9 +-------- tests/_helpers/helpers.js | 7 ++++--- tests/hijack/db.js | 4 ++-- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 3a18cb14..e9313b84 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ versions.json .idea .eslintcache .envrc +/packages/* \ No newline at end of file diff --git a/lib/hijack/async.js b/lib/hijack/async.js index eec19855..44beb6cd 100644 --- a/lib/hijack/async.js +++ b/lib/hijack/async.js @@ -1,5 +1,3 @@ -import { _ } from 'meteor/underscore'; - let Fibers = Npm.require('fibers'); let EventSymbol = Symbol('MontiEventSymbol'); let StartTracked = Symbol('MontiStartTracked'); @@ -26,16 +24,11 @@ export function wrapFibers () { wrapped = true; let originalYield = Fibers.yield; + Fibers.yield = function () { let kadiraInfo = Kadira._getInfo(); if (kadiraInfo) { - const lastEvent = _.last(kadiraInfo.trace.events); - - if (lastEvent.type && lastEvent.data && lastEvent.type === 'db' && lastEvent.data.func === 'fetch') { - return originalYield(); - } - let eventId = Kadira.tracer.event(kadiraInfo.trace, 'async'); if (eventId) { diff --git a/tests/_helpers/helpers.js b/tests/_helpers/helpers.js index 509c60dd..89104c3c 100644 --- a/tests/_helpers/helpers.js +++ b/tests/_helpers/helpers.js @@ -1,9 +1,10 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; import { DDP } from 'meteor/ddp'; -const Future = Npm.require('fibers/future'); import { MethodStore, TestData } from './globals'; +const Future = Npm.require('fibers/future'); + export const GetMeteorClient = function (_url) { const url = _url || Meteor.absoluteUrl(); return DDP.connect(url, {retry: false}); @@ -31,7 +32,7 @@ export const EnableTrackingMethods = function () { // }; }; -export const GetLastMethodEvents = function (_indices) { +export const GetLastMethodEvents = function (_indices, ignore = []) { if (MethodStore.length < 1) { return []; } @@ -43,7 +44,7 @@ export const GetLastMethodEvents = function (_indices) { return events; function isNotCompute (event) { - return event[0] !== 'compute'; + return event[0] !== 'compute' && !ignore.includes(event[0]); } function filterFields (event) { diff --git a/tests/hijack/db.js b/tests/hijack/db.js index 7a02d25b..dda03aa7 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -138,7 +138,7 @@ Tinytest.add( }); let client = GetMeteorClient(); let result = client.call(methodId); - let events = GetLastMethodEvents([0, 2]); + let events = GetLastMethodEvents([0, 2], ['async']); let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], @@ -174,7 +174,7 @@ Tinytest.add( }); let client = GetMeteorClient(); let result = client.call(methodId); - let events = GetLastMethodEvents([0, 2]); + let events = GetLastMethodEvents([0, 2], ['async']); let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], From 14f58f8d1c3ae1c90513cf754afa89628b76ed5c Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Thu, 5 Oct 2023 12:26:48 -0400 Subject: [PATCH 15/50] activate redis oplog tests based on its environment variable --- package.js | 22 +++++++++++++++++++--- tests/hijack/redis_oplog.js | 3 +++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 tests/hijack/redis_oplog.js diff --git a/package.js b/package.js index 14699bef..5e9c1c12 100644 --- a/package.js +++ b/package.js @@ -30,7 +30,19 @@ Package.onUse(function (api) { }); Package.onTest(function (api) { + if (process.env.REDIS_OPLOG_SETTINGS) { + api.use([ + 'cultofcoders:redis-oplog', + 'disable-oplog' + ], ['server']); + + api.addFiles([ + 'tests/hijack/redis_oplog.js', + ], 'server'); + } + configurePackage(api, true); + api.use([ 'peerlibrary:reactive-publish', 'tinytest', @@ -42,14 +54,18 @@ Package.onTest(function (api) { 'tests/models/base_error.js' ], ['client', 'server']); + // helpers + api.addFiles([ + 'tests/_helpers/globals.js', + 'tests/_helpers/helpers.js', + 'tests/_helpers/init.js', + ], ['server']); + // common server api.addFiles([ 'tests/utils.js', 'tests/ntp.js', 'tests/jobs.js', - 'tests/_helpers/globals.js', - 'tests/_helpers/helpers.js', - 'tests/_helpers/init.js', 'tests/ping.js', 'tests/hijack/info.js', 'tests/hijack/user.js', diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js new file mode 100644 index 00000000..7709b0e1 --- /dev/null +++ b/tests/hijack/redis_oplog.js @@ -0,0 +1,3 @@ +Tinytest.only('Database - Redis Oplog - Cursor - observeChanges', function (test) { + console.log('redis oplog'); +}); From 5823f8d5579a4268e8a4822379750a2a5dc36c65 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Fri, 6 Oct 2023 10:45:56 -0400 Subject: [PATCH 16/50] add base test for redis oplog --- lib/hijack/wrap_observers.js | 28 +++++++++++++++++++--------- tests/hijack/redis_oplog.js | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 93e25f0d..cf43dd83 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -1,7 +1,5 @@ import { _ } from 'meteor/underscore'; import { Random } from 'meteor/random'; -import { Mongo } from 'meteor/mongo'; -import { Meteor } from 'meteor/meteor'; export function wrapOplogObserveDriver (proto) { // Track the polled documents. This is reflected to the RAM size and @@ -97,14 +95,19 @@ export function wrapOplogObserveDriver (proto) { }; } +export function getDummyCollectionName () { + const collection = new Meteor.Collection(`__dummy_coll_${Random.id()}`); + const handler = collection.find({}).observeChanges({ added: Function.prototype }); + handler.stop(); + return collection._name; +} + export function wrapRedisOplogObserveDriver (driver) { - const driverProto = driver.prototype; - const MongoColl = typeof Mongo !== 'undefined' ? Mongo.Collection : Meteor.Collection; - const collection = new MongoColl(`__dummy_coll_${Random.id()}`); + const collectionName = getDummyCollectionName(); const dummyDriver = new driver({ cursorDescription: { - collectionName: collection._name, + collectionName, selector: {}, options: {} }, @@ -118,6 +121,7 @@ export function wrapRedisOplogObserveDriver (driver) { let originalAdd = observableCollectionProto.add; let originalChange = observableCollectionProto.change; let originalRemove = observableCollectionProto.remove; + // Track the polled documents. This is reflect to the RAM size and // for the CPU usage directly @@ -127,13 +131,15 @@ export function wrapRedisOplogObserveDriver (driver) { let opts = this.cursorDescription.options; let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. - // But it is used in the initial add and syntetic mutations, so we use it to get initial adds. + // But it is used in the initial add and synthetic mutations, so we use it to get initial adds. if (this._ownerInfo) { + console.log('tracking live updates'); Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'polledFetches', docSize); } else { // If there is no ownerInfo, that means this is the initial adds + // Also means it is not coming from a subscription if (!this._liveUpdatesCounts) { this._liveUpdatesCounts = { _initialAdds: 0 @@ -185,6 +191,7 @@ export function wrapRedisOplogObserveDriver (driver) { }; const originalRedisProcess = redisSubscriberProto.process; + redisSubscriberProto.process = function (op, doc, fields) { Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { op: redisEventToOplogMap[op] @@ -192,8 +199,9 @@ export function wrapRedisOplogObserveDriver (driver) { originalRedisProcess.call(this, op, doc, fields); }; - let originalStop = driverProto.stop; - driverProto.stop = function () { + // @todo check this + let originalStop = driver.prototype.stop; + driver.prototype.stop = function () { if (this.observableCollection._ownerInfo && this.observableCollection._ownerInfo.type === 'sub') { Kadira.EventBus.emit('pubsub', 'observerDeleted', this.observableCollection._ownerInfo); Kadira.models.pubsub.trackDeletedObserver(this.observableCollection._ownerInfo); @@ -290,6 +298,8 @@ export function wrapForCountingObservers () { // We store counts for redis-oplog in the observableCollection instead let ownerStorer = observerDriver.observableCollection || observerDriver; + console.log({ ownerInfo }); + ownerStorer._ownerInfo = ownerInfo; Kadira.EventBus.emit('pubsub', 'observerCreated', ownerInfo); diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index 7709b0e1..de0bd5dc 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -1,3 +1,32 @@ -Tinytest.only('Database - Redis Oplog - Cursor - observeChanges', function (test) { - console.log('redis oplog'); +import { TestData } from '../_helpers/globals'; +import { GetMeteorClient, RegisterPublication, SubscribeAndWait } from '../_helpers/helpers'; +import { prettyLog } from '../_helpers/pretty-log'; + +/** + * We only track the observers coming from subscriptions (which have `ownerInfo`) + */ +Tinytest.add('Database - Redis Oplog - Added', function (test) { + const pub = RegisterPublication(() => TestData.find({})); + + TestData.remove({}); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + + TestData.insert({ name: 'test1' }); + TestData.insert({ name: 'test2' }); + TestData.insert({ name: 'test3' }); + + Meteor._sleepForMs(100); + + const metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.totalObservers, 1); + test.equal(metrics.oplogInsertedDocuments, 3); + test.equal(metrics.liveAddedDocuments, 3); + + prettyLog(metrics); + + sub.stop(); + TestData.remove({}); }); From 7006c4482e03995daf6b7097ac2ea912139f2e36 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 10:52:38 -0400 Subject: [PATCH 17/50] fix running from meteor checkout --- tests/_helpers/helpers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_helpers/helpers.js b/tests/_helpers/helpers.js index 89104c3c..62e11dcc 100644 --- a/tests/_helpers/helpers.js +++ b/tests/_helpers/helpers.js @@ -174,7 +174,7 @@ export const WithDocCacheGetSize = function (fn, patchedSize) { } }; -export const releaseParts = Meteor.release.split('METEOR@')[1].split('.').map(num => parseInt(num, 10)); +export const releaseParts = (Meteor.release === 'none' ? 'METEOR@0.0.0' : Meteor.release).split('METEOR@')[1].split('.').map(num => parseInt(num, 10)); export const withRoundedTime = (fn) => (test) => { const date = new Date(); From c9f152743eb45f3a041ea8ca59ee9cd7cde18e6e Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 10:52:53 -0400 Subject: [PATCH 18/50] fix hijacks not working --- package.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.js b/package.js index 5e9c1c12..02855e46 100644 --- a/package.js +++ b/package.js @@ -30,6 +30,8 @@ Package.onUse(function (api) { }); Package.onTest(function (api) { + configurePackage(api, true); + if (process.env.REDIS_OPLOG_SETTINGS) { api.use([ 'cultofcoders:redis-oplog', @@ -41,8 +43,6 @@ Package.onTest(function (api) { ], 'server'); } - configurePackage(api, true); - api.use([ 'peerlibrary:reactive-publish', 'tinytest', From 3f33f48a221c03536724b282d046bb8fc414d253 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 10:53:25 -0400 Subject: [PATCH 19/50] this seems simpler --- lib/hijack/db.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index 202c0633..54332036 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -46,7 +46,8 @@ function getSyncronousCursor () { } export function hijackDBOps () { - let mongoConnectionProto = MeteorX.MongoConnection.prototype; + let mongoConnectionProto = MongoInternals.Connection.prototype; + // findOne is handled by find - so no need to track it // upsert is handles by update // 2.4 replaced _ensureIndex with createIndex From 263fde3dae2bc5b5e7e9e82b4a0e660282ebbebd Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 12:36:21 -0400 Subject: [PATCH 20/50] ignore async when redis oplog is active --- tests/_helpers/helpers.js | 2 ++ tests/hijack/db.js | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/_helpers/helpers.js b/tests/_helpers/helpers.js index 62e11dcc..5f3e2f90 100644 --- a/tests/_helpers/helpers.js +++ b/tests/_helpers/helpers.js @@ -194,6 +194,8 @@ export function addTestWithRoundedTime (name, fn) { Tinytest.add(name, withRoundedTime(fn)); } +export const isRedisOplogEnabled = !!process.env.REDIS_OPLOG_SETTINGS; + export const TestHelpers = { methodStore: MethodStore, getLatestEventsFromMethodStore: () => MethodStore[MethodStore.length - 1].events, diff --git a/tests/hijack/db.js b/tests/hijack/db.js index dda03aa7..c782bb7e 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -4,6 +4,7 @@ import { EnableTrackingMethods, GetLastMethodEvents, GetMeteorClient, + isRedisOplogEnabled, RegisterMethod } from '../_helpers/helpers'; @@ -45,7 +46,11 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'insert', async: true}], + ['db',undefined,{ + coll: 'tinytest-data', + func: 'insert', + ...!isRedisOplogEnabled ? { async: true } : {} + }], ['complete'] ]; test.equal(events, expected); From 48da81f18d1176bd4775e4d000106de3ac959dff Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 12:53:52 -0400 Subject: [PATCH 21/50] fix tests when running redis oplog --- tests/hijack/db.js | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/hijack/db.js b/tests/hijack/db.js index c782bb7e..f5745b7b 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -7,6 +7,7 @@ import { isRedisOplogEnabled, RegisterMethod } from '../_helpers/helpers'; +import { diffObjects } from '../_helpers/pretty-log'; Tinytest.add( 'Database - insert', @@ -102,7 +103,10 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined, {coll: 'tinytest-data', func: 'update', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}], + ... isRedisOplogEnabled ? [ + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docSize: 12, docsFetched: 1, limit: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'update', selector: JSON.stringify({_id: { $in: ['aa']}}), updatedDocs: 1}] + ] : [['db',undefined, {coll: 'tinytest-data', func: 'update', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}]], ['complete'] ]; test.equal(events, expected); @@ -125,7 +129,10 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined, {coll: 'tinytest-data', func: 'remove', selector: JSON.stringify({_id: 'aa'}), removedDocs: 1}], + ...isRedisOplogEnabled ? [ + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docSize: 12, docsFetched: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'remove', selector: JSON.stringify({_id: 'aa'}), removedDocs: 1}] + ] : [['db',undefined, {coll: 'tinytest-data', func: 'remove', selector: JSON.stringify({_id: 'aa'}), removedDocs: 1}]], ['complete'] ]; test.equal(events, expected); @@ -225,11 +232,20 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}], + ...isRedisOplogEnabled ? [ + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 0, docSize: 0, limit: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 1, docSize: 12, limit: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}], + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 1, docSize: 20 }] + ] : [ + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}] + ], ['complete'] ]; test.equal(events, expected); + diffObjects(events, expected); CleanTestData(); } ); @@ -249,8 +265,16 @@ Tinytest.add( let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}], - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}], + ...isRedisOplogEnabled ? [ + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 0, docSize: 0, limit: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 1, docSize: 12, limit: 1, projection: JSON.stringify({_id: 1})}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}], + ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 1, docSize: 20 }] + ] : [ + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}] + ], ['complete'] ]; test.equal(events, expected); From f07ba4783824e27ac5e7b20a56a8e3e4774d8861 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 9 Oct 2023 13:13:13 -0400 Subject: [PATCH 22/50] should track live instead of polled --- lib/hijack/wrap_observers.js | 3 +-- lib/models/pubsub.js | 7 ++++--- tests/_helpers/helpers.js | 4 ++++ tests/models/pubsub.js | 9 ++++++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index cf43dd83..9edcf637 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -133,10 +133,9 @@ export function wrapRedisOplogObserveDriver (driver) { // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. // But it is used in the initial add and synthetic mutations, so we use it to get initial adds. if (this._ownerInfo) { - console.log('tracking live updates'); Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); - Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'polledFetches', docSize); + Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'liveFetches', docSize); } else { // If there is no ownerInfo, that means this is the initial adds // Also means it is not coming from a subscription diff --git a/lib/models/pubsub.js b/lib/models/pubsub.js index b6c52ecc..3f723f71 100644 --- a/lib/models/pubsub.js +++ b/lib/models/pubsub.js @@ -1,11 +1,12 @@ import { Meteor } from 'meteor/meteor'; import { _ } from 'meteor/underscore'; -const logger = Npm.require('debug')('kadira:pubsub'); -const { DDSketch } = require('monti-apm-sketches-js'); import { KadiraModel } from './0model'; import { TracerStore } from '../tracer/tracer_store'; import { Ntp } from '../ntp'; -import {countKeys, getProperty, iterate} from '../utils'; +import { countKeys, getProperty, iterate } from '../utils'; + +const logger = Npm.require('debug')('kadira:pubsub'); +const { DDSketch } = require('monti-apm-sketches-js'); export function PubsubModel () { this.metricsByMinute = Object.create(null); diff --git a/tests/_helpers/helpers.js b/tests/_helpers/helpers.js index 5f3e2f90..065fe746 100644 --- a/tests/_helpers/helpers.js +++ b/tests/_helpers/helpers.js @@ -194,6 +194,10 @@ export function addTestWithRoundedTime (name, fn) { Tinytest.add(name, withRoundedTime(fn)); } +addTestWithRoundedTime.only = (name, fn) => { + Tinytest.only(name, withRoundedTime(fn), true); +}; + export const isRedisOplogEnabled = !!process.env.REDIS_OPLOG_SETTINGS; export const TestHelpers = { diff --git a/tests/models/pubsub.js b/tests/models/pubsub.js index b3379192..a7ecccbe 100644 --- a/tests/models/pubsub.js +++ b/tests/models/pubsub.js @@ -3,11 +3,13 @@ import { addTestWithRoundedTime, CleanTestData, CloseClient, - GetMeteorClient, GetPubSubPayload, + GetMeteorClient, + GetPubSubPayload, SubscribeAndWait, - Wait, WithDocCacheGetSize + Wait, + WithDocCacheGetSize } from '../_helpers/helpers'; -import {TestData} from '../_helpers/globals'; +import { TestData } from '../_helpers/globals'; addTestWithRoundedTime( 'Models - PubSub - Metrics - same date', @@ -671,6 +673,7 @@ addTestWithRoundedTime( }, 25); let payload = GetPubSubPayload(); + test.equal(payload[0].pubs['tinytest-data-random'].liveFetchedDocSize, 50); CloseClient(client); } From b521dcb9cecc97ed10ae83c709b760b534923dd2 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Tue, 10 Oct 2023 10:18:13 -0400 Subject: [PATCH 23/50] remove call from tests --- tests/hijack/db.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/hijack/db.js b/tests/hijack/db.js index f5745b7b..7885691c 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -7,7 +7,6 @@ import { isRedisOplogEnabled, RegisterMethod } from '../_helpers/helpers'; -import { diffObjects } from '../_helpers/pretty-log'; Tinytest.add( 'Database - insert', @@ -245,7 +244,6 @@ Tinytest.add( ['complete'] ]; test.equal(events, expected); - diffObjects(events, expected); CleanTestData(); } ); From d57690e095807063b7028c29a84bf40dcda8599e Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Tue, 10 Oct 2023 12:21:59 -0400 Subject: [PATCH 24/50] add test for removed docs --- tests/hijack/redis_oplog.js | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index de0bd5dc..7c66afd5 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -1,6 +1,5 @@ import { TestData } from '../_helpers/globals'; import { GetMeteorClient, RegisterPublication, SubscribeAndWait } from '../_helpers/helpers'; -import { prettyLog } from '../_helpers/pretty-log'; /** * We only track the observers coming from subscriptions (which have `ownerInfo`) @@ -25,7 +24,41 @@ Tinytest.add('Database - Redis Oplog - Added', function (test) { test.equal(metrics.oplogInsertedDocuments, 3); test.equal(metrics.liveAddedDocuments, 3); - prettyLog(metrics); + sub.stop(); + TestData.remove({}); +}); + +Tinytest.add('Database - Redis Oplog - Removed', function (test) { + const pub = RegisterPublication(() => TestData.find({})); + + TestData.remove({}); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + + TestData.insert({ name: 'test1' }); + TestData.insert({ name: 'test2' }); + TestData.insert({ name: 'test3' }); + + Meteor._sleepForMs(100); + + TestData.remove({ name: 'test2' }); + + Meteor._sleepForMs(100); + + let metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.totalObservers, 1); + test.equal(metrics.liveRemovedDocuments, 1); + + TestData.remove({}); + + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.totalObservers, 1); + test.equal(metrics.liveRemovedDocuments, 3); sub.stop(); TestData.remove({}); From 9996bdd7e13570aa18e7686b89dab6b7d828a2e4 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Tue, 10 Oct 2023 12:23:56 -0400 Subject: [PATCH 25/50] add test for updated --- tests/hijack/redis_oplog.js | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index 7c66afd5..79618ef2 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -63,3 +63,39 @@ Tinytest.add('Database - Redis Oplog - Removed', function (test) { sub.stop(); TestData.remove({}); }); + +Tinytest.add('Database - Redis Oplog - Changed', function (test) { + const pub = RegisterPublication(() => TestData.find({})); + + TestData.remove({}); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + + TestData.insert({ name: 'test1' }); + TestData.insert({ name: 'test2' }); + TestData.insert({ name: 'test3' }); + + Meteor._sleepForMs(100); + + TestData.update({ name: 'test2' }, { $set: { name: 'test4' } }); + + Meteor._sleepForMs(100); + + let metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.totalObservers, 1); + test.equal(metrics.liveChangedDocuments, 1); + + TestData.update({}, { $set: { name: 'test5' } }, { multi: true }); + + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.totalObservers, 1); + test.equal(metrics.liveChangedDocuments, 4); + + sub.stop(); + TestData.remove({}); +}); From 5ffafbe74dff86bf84323f58267eb28225e3b2bf Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 11 Oct 2023 09:05:48 -0400 Subject: [PATCH 26/50] fix test --- tests/hijack/db.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/hijack/db.js b/tests/hijack/db.js index 7885691c..3120ffc0 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -270,8 +270,8 @@ Tinytest.add( ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}], ['db',undefined,{coll: 'tinytest-data', func: 'fetch', selector: JSON.stringify({_id: 'aa'}), cursor: true, docsFetched: 1, docSize: 20 }] ] : [ - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: 'aa'}], - ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1, insertedId: undefined}] + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}], + ['db',undefined,{coll: 'tinytest-data', func: 'upsert', selector: JSON.stringify({_id: 'aa'}), updatedDocs: 1}] ], ['complete'] ]; From c085f0f8a5b5932989bad5f921e27c908299768b Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 11 Oct 2023 09:51:07 -0400 Subject: [PATCH 27/50] add tests for redis oplog --- .github/workflows/test-redis-oplog.yml | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/test-redis-oplog.yml diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml new file mode 100644 index 00000000..44145dae --- /dev/null +++ b/.github/workflows/test-redis-oplog.yml @@ -0,0 +1,38 @@ +name: Test Redis Oplog +on: + push: + branches: + - master + pull_request: +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + meteorRelease: + - '--release 1.4.4.6' + - '--release 1.12.1' + - '--release 2.1.1' + # Latest version + - + env: + REDIS_OPLOG_SETTINGS: '{"debug":true}' + steps: + - uses: supercharge/redis-github-action@1.4.0 + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: '12.x' + - name: Install Dependencies + run: | + curl https://install.meteor.com | /bin/sh + npm i -g @zodern/mtest + + - name: Run Tests + run: | + # Fix using old versions of Meteor + export NODE_TLS_REJECT_UNAUTHORIZED=0 + + mtest --package ./ --once ${{ matrix.meteorRelease }} From fcd1f40983b92cb1139574c6a25ce35654f543ce Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 11 Oct 2023 13:17:49 -0400 Subject: [PATCH 28/50] move redis oplog registration --- package.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/package.js b/package.js index 02855e46..c5c8bd9f 100644 --- a/package.js +++ b/package.js @@ -32,17 +32,6 @@ Package.onUse(function (api) { Package.onTest(function (api) { configurePackage(api, true); - if (process.env.REDIS_OPLOG_SETTINGS) { - api.use([ - 'cultofcoders:redis-oplog', - 'disable-oplog' - ], ['server']); - - api.addFiles([ - 'tests/hijack/redis_oplog.js', - ], 'server'); - } - api.use([ 'peerlibrary:reactive-publish', 'tinytest', @@ -196,6 +185,17 @@ function configurePackage (api, isTesting) { 'lib/conflicting_agents.js', ], 'server'); + if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { + api.use([ + 'cultofcoders:redis-oplog', + 'disable-oplog' + ], ['server']); + + api.addFiles([ + 'tests/hijack/redis_oplog.js', + ], 'server'); + } + // only client api.addFiles([ 'lib/retry.js', From ec6b669fc6c13222ccd71666a7717e0a303e8d3c Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 11 Oct 2023 13:24:44 -0400 Subject: [PATCH 29/50] add delay so it doesn't affect other tests --- tests/hijack/redis_oplog.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index 79618ef2..d1bb961f 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -26,6 +26,8 @@ Tinytest.add('Database - Redis Oplog - Added', function (test) { sub.stop(); TestData.remove({}); + + Meteor._sleepForMs(100); }); Tinytest.add('Database - Redis Oplog - Removed', function (test) { @@ -62,6 +64,8 @@ Tinytest.add('Database - Redis Oplog - Removed', function (test) { sub.stop(); TestData.remove({}); + + Meteor._sleepForMs(100); }); Tinytest.add('Database - Redis Oplog - Changed', function (test) { @@ -98,4 +102,6 @@ Tinytest.add('Database - Redis Oplog - Changed', function (test) { sub.stop(); TestData.remove({}); + + Meteor._sleepForMs(100); }); From 9ebcf87c77518e3e8f4823fdd8475084c6bba223 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Thu, 12 Oct 2023 13:02:07 -0400 Subject: [PATCH 30/50] set redis host --- .github/workflows/test-redis-oplog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml index 44145dae..7397c20c 100644 --- a/.github/workflows/test-redis-oplog.yml +++ b/.github/workflows/test-redis-oplog.yml @@ -18,7 +18,7 @@ jobs: # Latest version - env: - REDIS_OPLOG_SETTINGS: '{"debug":true}' + REDIS_OPLOG_SETTINGS: '{"redis":{"port": 6379,"host": "redis"},"debug":true}' steps: - uses: supercharge/redis-github-action@1.4.0 - uses: actions/checkout@v2 From cc951dd5e56613ded09d3021b2b68bfbb679479a Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 16 Oct 2023 14:49:08 -0400 Subject: [PATCH 31/50] add versions to packages --- package.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.js b/package.js index c5c8bd9f..ad1142e4 100644 --- a/package.js +++ b/package.js @@ -187,8 +187,8 @@ function configurePackage (api, isTesting) { if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { api.use([ - 'cultofcoders:redis-oplog', - 'disable-oplog' + 'cultofcoders:redis-oplog@2.2.1', + 'disable-oplog@1.0.7' ], ['server']); api.addFiles([ From 22657914aac077df76c6d25ea82dd488cfc3679c Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 16 Oct 2023 14:57:51 -0400 Subject: [PATCH 32/50] add v1 too --- package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.js b/package.js index ad1142e4..0533411e 100644 --- a/package.js +++ b/package.js @@ -187,7 +187,7 @@ function configurePackage (api, isTesting) { if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { api.use([ - 'cultofcoders:redis-oplog@2.2.1', + 'cultofcoders:redis-oplog@1.0.14||2.2.1', 'disable-oplog@1.0.7' ], ['server']); From c03273a242e7ee62323d9534c57cd0c4c751d943 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Mon, 16 Oct 2023 15:14:28 -0400 Subject: [PATCH 33/50] fix flaky test --- tests/hijack/db.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/hijack/db.js b/tests/hijack/db.js index 3120ffc0..09cf1c32 100644 --- a/tests/hijack/db.js +++ b/tests/hijack/db.js @@ -286,13 +286,18 @@ Tinytest.add( EnableTrackingMethods(); let name = typeof TestData.createIndex === 'function' ? 'createIndex' : '_ensureIndex'; let methodId = RegisterMethod(function () { - TestData[name]({aa: 1, bb: 1}); - TestData._dropIndex({aa: 1, bb: 1}); - return 'indexes'; + try { + TestData[name]({aa: 1, bb: 1}); + Meteor._sleepForMs(100); + TestData._dropIndex({aa: 1, bb: 1}); + return 'indexes'; + } catch (e) { + console.error(e); + } }); let client = GetMeteorClient(); client.call(methodId); - let events = GetLastMethodEvents([0, 2]); + let events = GetLastMethodEvents([0, 2], ['async']); let expected = [ ['start',undefined,{userId: null, params: '[]'}], ['wait',undefined,{waitOn: []}], From fba613ac3b7799020aed6e3bde4e169de56cfdcb Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Tue, 17 Oct 2023 09:03:48 -0400 Subject: [PATCH 34/50] move package registration to the top and remove redis host --- .github/workflows/test-redis-oplog.yml | 2 +- package.js | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml index 7397c20c..44145dae 100644 --- a/.github/workflows/test-redis-oplog.yml +++ b/.github/workflows/test-redis-oplog.yml @@ -18,7 +18,7 @@ jobs: # Latest version - env: - REDIS_OPLOG_SETTINGS: '{"redis":{"port": 6379,"host": "redis"},"debug":true}' + REDIS_OPLOG_SETTINGS: '{"debug":true}' steps: - uses: supercharge/redis-github-action@1.4.0 - uses: actions/checkout@v2 diff --git a/package.js b/package.js index 0533411e..bb235b5e 100644 --- a/package.js +++ b/package.js @@ -124,6 +124,14 @@ function canRunTestsWithFetch () { function configurePackage (api, isTesting) { api.versionsFrom('METEOR@1.4'); + + if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { + api.use([ + 'cultofcoders:redis-oplog@1.0.14||2.2.1', + 'disable-oplog@1.0.7' + ], ['server']); + } + api.use('montiapm:meteorx@2.2.0', ['server']); api.use('meteorhacks:zones@1.2.1', { weak: true }); api.use('simple:json-routes@2.1.0', { weak: true }); @@ -186,11 +194,6 @@ function configurePackage (api, isTesting) { ], 'server'); if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { - api.use([ - 'cultofcoders:redis-oplog@1.0.14||2.2.1', - 'disable-oplog@1.0.7' - ], ['server']); - api.addFiles([ 'tests/hijack/redis_oplog.js', ], 'server'); From 4fd2d70d9090730e5210bacea1cf0b65be203545 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Tue, 17 Oct 2023 09:55:13 -0400 Subject: [PATCH 35/50] remove v1 --- package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.js b/package.js index bb235b5e..3f9b3100 100644 --- a/package.js +++ b/package.js @@ -127,7 +127,7 @@ function configurePackage (api, isTesting) { if (isTesting && process.env.REDIS_OPLOG_SETTINGS) { api.use([ - 'cultofcoders:redis-oplog@1.0.14||2.2.1', + 'cultofcoders:redis-oplog@2.2.1', 'disable-oplog@1.0.7' ], ['server']); } From 2ea04e8d66c727284563af51718a006660002396 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 07:46:44 -0400 Subject: [PATCH 36/50] standardize projection field --- lib/hijack/db.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/hijack/db.js b/lib/hijack/db.js index 54332036..1b3d43b2 100644 --- a/lib/hijack/db.js +++ b/lib/hijack/db.js @@ -142,7 +142,16 @@ export function hijackDBOps () { }); if (cursorDescription.options) { - let cursorOptions = _.pick(cursorDescription.options, ['fields', 'projection', 'sort', 'limit']); + const opts = cursorDescription.options; + + const projection = opts.fields || opts.projection; + + let cursorOptions = _.pick(cursorDescription.options, ['sort', 'limit']); + + if (projection) { + cursorOptions.projection = projection; + } + for (let field in cursorOptions) { let value = cursorOptions[field]; if (typeof value === 'object') { From 05ea977438f854fecb8e18e62679a68bdf63a9d7 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 07:47:16 -0400 Subject: [PATCH 37/50] remove test for unsupported meteor version (redis oplog) --- .github/workflows/test-redis-oplog.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml index 44145dae..833cf010 100644 --- a/.github/workflows/test-redis-oplog.yml +++ b/.github/workflows/test-redis-oplog.yml @@ -12,7 +12,6 @@ jobs: fail-fast: false matrix: meteorRelease: - - '--release 1.4.4.6' - '--release 1.12.1' - '--release 2.1.1' # Latest version From 968287bd9ee6d43e6aa456054b25c49a094dcf18 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 08:08:39 -0400 Subject: [PATCH 38/50] fix flaky test --- tests/hijack/subscriptions.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/hijack/subscriptions.js b/tests/hijack/subscriptions.js index 900850d9..c3fba792 100644 --- a/tests/hijack/subscriptions.js +++ b/tests/hijack/subscriptions.js @@ -1,11 +1,17 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; import { - CleanTestData, CloseClient, - EnableTrackingMethods, FindMetricsForPub, - GetMeteorClient, GetPubSubMetrics, GetPubSubPayload, RegisterPublication, + CleanTestData, + CloseClient, + EnableTrackingMethods, + FindMetricsForPub, + GetMeteorClient, + GetPubSubMetrics, + GetPubSubPayload, + RegisterPublication, SubscribeAndWait, - TestHelpers, Wait + TestHelpers, + Wait } from '../_helpers/helpers'; Tinytest.add( @@ -107,7 +113,7 @@ Tinytest.add( h1.stop(); CloseClient(client); let metrics = FindMetricsForPub('tinytest-data'); - test.isTrue(TestHelpers.compareNear(metrics.lifeTime, 50, 75)); + test.isTrue(TestHelpers.compareNear(metrics.lifeTime, 50, 200)); } ); From 960c585e52a0b5ec4a314858b6e617cefa894993 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 08:13:39 -0400 Subject: [PATCH 39/50] add comment --- tests/hijack/subscriptions.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/hijack/subscriptions.js b/tests/hijack/subscriptions.js index c3fba792..afcdbfd5 100644 --- a/tests/hijack/subscriptions.js +++ b/tests/hijack/subscriptions.js @@ -102,6 +102,10 @@ Tinytest.add( // } // ); +/** + * The `lifeTime` metric seems flaky on Meteor v2.8.2 specifically, + * shouldn't be something worry about but leaving this comment just in case. + */ Tinytest.add( 'Subscriptions - Lifetime - sub', function (test) { From 41b6a852fcfe8dfec0e6a864c8cf4416ac69ad2d Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 09:14:30 -0400 Subject: [PATCH 40/50] test initially added documents --- tests/hijack/redis_oplog.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index d1bb961f..6916e96c 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -9,17 +9,23 @@ Tinytest.add('Database - Redis Oplog - Added', function (test) { TestData.remove({}); - const client = GetMeteorClient(); - const sub = SubscribeAndWait(client, pub); - TestData.insert({ name: 'test1' }); TestData.insert({ name: 'test2' }); TestData.insert({ name: 'test3' }); + TestData.insert({ name: 'test4' }); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + + TestData.insert({ name: 'test5' }); + TestData.insert({ name: 'test6' }); + TestData.insert({ name: 'test7' }); Meteor._sleepForMs(100); const metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + test.equal(metrics.initiallyAddedDocuments, 4); test.equal(metrics.totalObservers, 1); test.equal(metrics.oplogInsertedDocuments, 3); test.equal(metrics.liveAddedDocuments, 3); From 01ddd15ca6cf35a318926c19fe7e52f499297618 Mon Sep 17 00:00:00 2001 From: Leonardo Venturini Date: Wed, 18 Oct 2023 09:20:51 -0400 Subject: [PATCH 41/50] extract redis oplog wrapper into separate file --- lib/hijack/instrument.js | 2 +- lib/hijack/redis_oplog.js | 116 +++++++++++++++++++++++++++++++++ lib/hijack/wrap_observers.js | 120 +---------------------------------- package.js | 1 + 4 files changed, 120 insertions(+), 119 deletions(-) create mode 100644 lib/hijack/redis_oplog.js diff --git a/lib/hijack/instrument.js b/lib/hijack/instrument.js index ac54afa6..87d5d222 100644 --- a/lib/hijack/instrument.js +++ b/lib/hijack/instrument.js @@ -12,11 +12,11 @@ import { wrapMultiplexer, wrapOplogObserveDriver, wrapPollingObserveDriver, - wrapRedisOplogObserveDriver } from './wrap_observers'; import { wrapStringifyDDP } from './wrap_ddp_stringify'; import { setLabels } from './set_labels'; import { hijackDBOps } from './db'; +import { wrapRedisOplogObserveDriver } from './redis_oplog'; let instrumented = false; Kadira._startInstrumenting = function (callback) { diff --git a/lib/hijack/redis_oplog.js b/lib/hijack/redis_oplog.js new file mode 100644 index 00000000..f8e7a006 --- /dev/null +++ b/lib/hijack/redis_oplog.js @@ -0,0 +1,116 @@ +import { Random } from 'meteor/random'; + +export function getDummyCollectionName () { + const collection = new Meteor.Collection(`__dummy_coll_${Random.id()}`); + const handler = collection.find({}).observeChanges({ added: Function.prototype }); + handler.stop(); + return collection._name; +} + +export function wrapRedisOplogObserveDriver (driver) { + const collectionName = getDummyCollectionName(); + + const dummyDriver = new driver({ + cursorDescription: { + collectionName, + selector: {}, + options: {} + }, + multiplexer: { ready () {} }, + matcher: { combineIntoProjection: () => ({}) }, + }); + + const observableCollectionProto = dummyDriver.observableCollection.constructor.prototype; + const redisSubscriberProto = dummyDriver.redisSubscriber.constructor.prototype; + + let originalAdd = observableCollectionProto.add; + let originalChange = observableCollectionProto.change; + let originalRemove = observableCollectionProto.remove; + + // Track the polled documents. This is reflect to the RAM size and + // for the CPU usage directly + + observableCollectionProto.add = function (doc, safe) { + let coll = this.cursorDescription.collectionName; + let query = this.cursorDescription.selector; + let opts = this.cursorDescription.options; + let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); + // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. + // But it is used in the initial add and synthetic mutations, so we use it to get initial adds. + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); + Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); + Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'liveFetches', docSize); + } else { + // If there is no ownerInfo, that means this is the initial adds + // Also means it is not coming from a subscription + if (!this._liveUpdatesCounts) { + this._liveUpdatesCounts = { + _initialAdds: 0 + }; + } + + this._liveUpdatesCounts._initialAdds++; + + if (this._polledDocuments) { + this._polledDocuments += 1; + } else { + this._polledDocuments = 1; + } + + if (this._docSize) { + this._docSize.polledFetches += docSize; + } else { + this._docSize = { + polledFetches: docSize, + initialFetches: 0 + }; + } + + this._docSize.initialFetches += docSize; + } + + return originalAdd.call(this, doc, safe); + }; + + observableCollectionProto.change = function (doc, modifiedFields) { + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_changePublished', 1); + } + originalChange.call(this, doc, modifiedFields); + }; + + observableCollectionProto.remove = function (docId) { + if (this._ownerInfo) { + Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_removePublished', 1); + } + originalRemove.call(this, docId); + }; + + // Redis and oplog don't have the same op constants + const redisEventToOplogMap = { + i: 'i', + u: 'u', + r: 'd', + }; + + const originalRedisProcess = redisSubscriberProto.process; + + redisSubscriberProto.process = function (op, doc, fields) { + Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { + op: redisEventToOplogMap[op] + }); + originalRedisProcess.call(this, op, doc, fields); + }; + + // @todo check this + let originalStop = driver.prototype.stop; + driver.prototype.stop = function () { + if (this.observableCollection._ownerInfo && this.observableCollection._ownerInfo.type === 'sub') { + Kadira.EventBus.emit('pubsub', 'observerDeleted', this.observableCollection._ownerInfo); + Kadira.models.pubsub.trackDeletedObserver(this.observableCollection._ownerInfo); + } + + return originalStop.call(this); + }; +} diff --git a/lib/hijack/wrap_observers.js b/lib/hijack/wrap_observers.js index 9edcf637..e5a6450b 100644 --- a/lib/hijack/wrap_observers.js +++ b/lib/hijack/wrap_observers.js @@ -1,5 +1,4 @@ import { _ } from 'meteor/underscore'; -import { Random } from 'meteor/random'; export function wrapOplogObserveDriver (proto) { // Track the polled documents. This is reflected to the RAM size and @@ -95,121 +94,6 @@ export function wrapOplogObserveDriver (proto) { }; } -export function getDummyCollectionName () { - const collection = new Meteor.Collection(`__dummy_coll_${Random.id()}`); - const handler = collection.find({}).observeChanges({ added: Function.prototype }); - handler.stop(); - return collection._name; -} - -export function wrapRedisOplogObserveDriver (driver) { - const collectionName = getDummyCollectionName(); - - const dummyDriver = new driver({ - cursorDescription: { - collectionName, - selector: {}, - options: {} - }, - multiplexer: { ready () {} }, - matcher: { combineIntoProjection: () => ({}) }, - }); - - const observableCollectionProto = dummyDriver.observableCollection.constructor.prototype; - const redisSubscriberProto = dummyDriver.redisSubscriber.constructor.prototype; - - let originalAdd = observableCollectionProto.add; - let originalChange = observableCollectionProto.change; - let originalRemove = observableCollectionProto.remove; - - // Track the polled documents. This is reflect to the RAM size and - // for the CPU usage directly - - observableCollectionProto.add = function (doc, safe) { - let coll = this.cursorDescription.collectionName; - let query = this.cursorDescription.selector; - let opts = this.cursorDescription.options; - let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); - // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. - // But it is used in the initial add and synthetic mutations, so we use it to get initial adds. - if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); - Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); - Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'liveFetches', docSize); - } else { - // If there is no ownerInfo, that means this is the initial adds - // Also means it is not coming from a subscription - if (!this._liveUpdatesCounts) { - this._liveUpdatesCounts = { - _initialAdds: 0 - }; - } - - this._liveUpdatesCounts._initialAdds++; - - if (this._polledDocuments) { - this._polledDocuments += 1; - } else { - this._polledDocuments = 1; - } - - if (this._docSize) { - this._docSize.polledFetches += docSize; - } else { - this._docSize = { - polledFetches: docSize, - initialFetches: 0 - }; - } - - this._docSize.initialFetches += docSize; - } - - return originalAdd.call(this, doc, safe); - }; - - observableCollectionProto.change = function (doc, modifiedFields) { - if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_changePublished', 1); - } - originalChange.call(this, doc, modifiedFields); - }; - - observableCollectionProto.remove = function (docId) { - if (this._ownerInfo) { - Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_removePublished', 1); - } - originalRemove.call(this, docId); - }; - - // Redis and oplog don't have the same op constants - const redisEventToOplogMap = { - i: 'i', - u: 'u', - r: 'd', - }; - - const originalRedisProcess = redisSubscriberProto.process; - - redisSubscriberProto.process = function (op, doc, fields) { - Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { - op: redisEventToOplogMap[op] - }); - originalRedisProcess.call(this, op, doc, fields); - }; - - // @todo check this - let originalStop = driver.prototype.stop; - driver.prototype.stop = function () { - if (this.observableCollection._ownerInfo && this.observableCollection._ownerInfo.type === 'sub') { - Kadira.EventBus.emit('pubsub', 'observerDeleted', this.observableCollection._ownerInfo); - Kadira.models.pubsub.trackDeletedObserver(this.observableCollection._ownerInfo); - } - - return originalStop.call(this); - }; -} - export function wrapPollingObserveDriver (proto) { let originalPollMongo = proto._pollMongo; proto._pollMongo = function () { @@ -293,12 +177,12 @@ export function wrapForCountingObservers () { startTime: new Date().getTime() }; + console.log('ownerInfo', ownerInfo); + let observerDriver = ret._multiplexer._observeDriver; // We store counts for redis-oplog in the observableCollection instead let ownerStorer = observerDriver.observableCollection || observerDriver; - console.log({ ownerInfo }); - ownerStorer._ownerInfo = ownerInfo; Kadira.EventBus.emit('pubsub', 'observerCreated', ownerInfo); diff --git a/package.js b/package.js index 3f9b3100..0cc7dac6 100644 --- a/package.js +++ b/package.js @@ -188,6 +188,7 @@ function configurePackage (api, isTesting) { 'lib/hijack/async.js', 'lib/hijack/error.js', 'lib/hijack/set_labels.js', + 'lib/hijack/redis_oplog.js', 'lib/environment_variables.js', 'lib/auto_connect.js', 'lib/conflicting_agents.js', From 3f2ad5755a134c144f733a65ef73649160eb7351 Mon Sep 17 00:00:00 2001 From: Renan Date: Tue, 26 Dec 2023 18:25:38 -0300 Subject: [PATCH 42/50] feat: support skip/limit and requery/reload cases --- lib/hijack/redis_oplog.js | 23 ++++++++++++++++++++-- tests/hijack/redis_oplog.js | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/lib/hijack/redis_oplog.js b/lib/hijack/redis_oplog.js index f8e7a006..ef0c1526 100644 --- a/lib/hijack/redis_oplog.js +++ b/lib/hijack/redis_oplog.js @@ -7,6 +7,16 @@ export function getDummyCollectionName () { return collection._name; } +function rewriteReloadRequeryFuncs () { + // handle reload/requery cases + const originalCompareWith = this.store.constructor.prototype.compareWith; + this.store.constructor.prototype.compareWith = (other, callbacks) => { + Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, other.size()); + + return originalCompareWith.call(this.store, other, callbacks); + }; +} + export function wrapRedisOplogObserveDriver (driver) { const collectionName = getDummyCollectionName(); @@ -30,13 +40,16 @@ export function wrapRedisOplogObserveDriver (driver) { // Track the polled documents. This is reflect to the RAM size and // for the CPU usage directly + + // According to the comments in redis-oplog, the "safe" param means that the document is "cleaned". + // it is set to true in the initial add and synthetic mutations observableCollectionProto.add = function (doc, safe) { + // handle reload/requery cases + rewriteReloadRequeryFuncs.call(this); let coll = this.cursorDescription.collectionName; let query = this.cursorDescription.selector; let opts = this.cursorDescription.options; let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); - // According to the comments in redis-oplog, "safe" means that the document is "cleaned", whatever that means. - // But it is used in the initial add and synthetic mutations, so we use it to get initial adds. if (this._ownerInfo) { Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); @@ -74,6 +87,9 @@ export function wrapRedisOplogObserveDriver (driver) { }; observableCollectionProto.change = function (doc, modifiedFields) { + // handle reload/requery cases + rewriteReloadRequeryFuncs.call(this); + if (this._ownerInfo) { Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_changePublished', 1); } @@ -81,6 +97,9 @@ export function wrapRedisOplogObserveDriver (driver) { }; observableCollectionProto.remove = function (docId) { + // handle reload/requery cases + rewriteReloadRequeryFuncs.call(this); + if (this._ownerInfo) { Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_removePublished', 1); } diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index 6916e96c..c0bde591 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -36,6 +36,44 @@ Tinytest.add('Database - Redis Oplog - Added', function (test) { Meteor._sleepForMs(100); }); +Tinytest.add('Database - Redis Oplog - Added with limit/skip', function (test) { + const pub = RegisterPublication(() => TestData.find({name: 'test'}, {limit: 2, skip: 0})); + + TestData.remove({}); + + TestData.insert({ name: 'test' }); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + let metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.polledDocuments, 1); + + TestData.insert({ name: 'test' }); + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + test.equal(metrics.polledDocuments, 1); + + TestData.insert({ name: 'doesnt-match-cursor' }); + // as the selector is not matched, redis-oplog triggers a requery + + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.initiallyAddedDocuments, 1); + test.equal(metrics.totalObservers, 1); + test.equal(metrics.oplogInsertedDocuments, 2); + test.equal(metrics.liveAddedDocuments, 1); + // 1 from initial poll + 3 from last requery + test.equal(metrics.polledDocuments, 4); + + + sub.stop(); + TestData.remove({}); + + Meteor._sleepForMs(100); +}); + Tinytest.add('Database - Redis Oplog - Removed', function (test) { const pub = RegisterPublication(() => TestData.find({})); From 7991d47851aa57327591d8400a5ae1c3bd6f3e78 Mon Sep 17 00:00:00 2001 From: Renan Date: Wed, 27 Dec 2023 16:12:53 -0300 Subject: [PATCH 43/50] feat: support protectAgainstRaceConditions option and differentiate between them --- lib/hijack/redis_oplog.js | 19 +++++++- tests/_helpers/globals.js | 4 ++ tests/hijack/redis_oplog.js | 92 +++++++++++++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/lib/hijack/redis_oplog.js b/lib/hijack/redis_oplog.js index ef0c1526..e7220ccf 100644 --- a/lib/hijack/redis_oplog.js +++ b/lib/hijack/redis_oplog.js @@ -6,6 +6,16 @@ export function getDummyCollectionName () { handler.stop(); return collection._name; } +function protectAgainstRaceConditions (collection) { + if (!collection._redisOplog) { + return true; + } + + return ( + collection._redisOplog && + collection._redisOplog.protectAgainstRaceConditions + ); +} function rewriteReloadRequeryFuncs () { // handle reload/requery cases @@ -52,7 +62,6 @@ export function wrapRedisOplogObserveDriver (driver) { let docSize = Kadira.docSzCache.getSize(coll, query, opts, [doc]); if (this._ownerInfo) { Kadira.models.pubsub.trackLiveUpdates(this._ownerInfo, '_addPublished', 1); - Kadira.models.pubsub.trackPolledDocuments(this._ownerInfo, 1); Kadira.models.pubsub.trackDocSize(this._ownerInfo.name, 'liveFetches', docSize); } else { // If there is no ownerInfo, that means this is the initial adds @@ -119,6 +128,14 @@ export function wrapRedisOplogObserveDriver (driver) { Kadira.models.pubsub.trackDocumentChanges(this.observableCollection._ownerInfo, { op: redisEventToOplogMap[op] }); + const collection = this.observableCollection.collection; + // redis-oplog always fetches the document again, except when: + // - operator === removal + // - there is an explicit _redisOplog config passed with collection.configureRedisOplog and that includes protectAgainstRaceConditions = false + // in this case there is a fetch counted in the collection level, not in the publication level as it happens while doing the insert + if (op !== 'r' && protectAgainstRaceConditions(collection)) { + Kadira.models.pubsub.trackPolledDocuments(this.observableCollection._ownerInfo, 1); + } originalRedisProcess.call(this, op, doc, fields); }; diff --git a/tests/_helpers/globals.js b/tests/_helpers/globals.js index 76b37a83..34fa13ca 100644 --- a/tests/_helpers/globals.js +++ b/tests/_helpers/globals.js @@ -2,3 +2,7 @@ import { Meteor } from 'meteor/meteor'; export const MethodStore = []; export const TestData = new Meteor.Collection('tinytest-data'); +export const TestDataRedis = new Meteor.Collection('tinytest-data-redis'); +TestDataRedis.configureRedisOplog({}); +export const TestDataRedisNoRaceProtection = new Meteor.Collection('tinytest-data-redis-NoRaceProtection'); +TestDataRedisNoRaceProtection.configureRedisOplog({protectAgainstRaceConditions: false}); diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index c0bde591..56e16547 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -1,4 +1,4 @@ -import { TestData } from '../_helpers/globals'; +import { TestData, TestDataRedis, TestDataRedisNoRaceProtection } from '../_helpers/globals'; import { GetMeteorClient, RegisterPublication, SubscribeAndWait } from '../_helpers/helpers'; /** @@ -50,11 +50,14 @@ Tinytest.add('Database - Redis Oplog - Added with limit/skip', function (test) { test.equal(metrics.polledDocuments, 1); TestData.insert({ name: 'test' }); + Meteor._sleepForMs(100); + // as the selector IS matched, redis-oplog triggers a requery metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); - test.equal(metrics.polledDocuments, 1); + // 1 from initial subscription, 1 findOne + requery(2) + test.equal(metrics.polledDocuments, 4); TestData.insert({ name: 'doesnt-match-cursor' }); - // as the selector is not matched, redis-oplog triggers a requery + // as the selector IS NOT matched, redis-oplog does not trigger a requery Meteor._sleepForMs(100); @@ -64,8 +67,8 @@ Tinytest.add('Database - Redis Oplog - Added with limit/skip', function (test) { test.equal(metrics.totalObservers, 1); test.equal(metrics.oplogInsertedDocuments, 2); test.equal(metrics.liveAddedDocuments, 1); - // 1 from initial poll + 3 from last requery - test.equal(metrics.polledDocuments, 4); + // 4 from before + 1 findOne from the unmatched document + test.equal(metrics.polledDocuments, 5); sub.stop(); @@ -74,6 +77,85 @@ Tinytest.add('Database - Redis Oplog - Added with limit/skip', function (test) { Meteor._sleepForMs(100); }); +Tinytest.add('Database - Redis Oplog - With protect against race condition', function (test) { + // in this case, every subscriber will refetch the doc once when receiving it + const pub = RegisterPublication(() => TestDataRedis.find({name: 'test'})); + + TestDataRedis.remove({}); + + TestDataRedis.insert({ name: 'test' }); + + const client = GetMeteorClient(); + const client2 = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + const sub2 = SubscribeAndWait(client2, pub); + let metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.polledDocuments, 1); + + TestDataRedis.insert({ name: 'test' }); + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + test.equal(metrics.polledDocuments, 2); + + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.initiallyAddedDocuments, 1); + test.equal(metrics.totalObservers, 2); + test.equal(metrics.oplogInsertedDocuments, 1); + test.equal(metrics.liveAddedDocuments, 1); + test.equal(metrics.polledDocuments, 2); + + + sub.stop(); + sub2.stop(); + TestDataRedis.remove({}); + + Meteor._sleepForMs(100); +}); + +Tinytest.add('Database - Redis Oplog - Without protect against race condition', function (test) { + // in this case, no subscriber will refetch the doc when receiving it + const pub = RegisterPublication(() => TestDataRedisNoRaceProtection.find({})); + + TestDataRedisNoRaceProtection.remove({}); + + TestDataRedisNoRaceProtection.insert({ name: 'test' }); + + const client = GetMeteorClient(); + const sub = SubscribeAndWait(client, pub); + const sub2 = SubscribeAndWait(client, pub); + let metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.polledDocuments, 1); + + TestDataRedisNoRaceProtection.insert({ name: 'test' }); + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + test.equal(metrics.polledDocuments, 1); + + Meteor._sleepForMs(100); + + metrics = Kadira.models.pubsub._getMetrics(new Date(), pub); + + test.equal(metrics.initiallyAddedDocuments, 1); + test.equal(metrics.totalObservers, 2); + test.equal(metrics.oplogInsertedDocuments, 1); + test.equal(metrics.liveAddedDocuments, 1); + test.equal(metrics.polledDocuments, 1); + + + sub.stop(); + sub2.stop(); + TestDataRedisNoRaceProtection.remove({}); + + Meteor._sleepForMs(100); +}); + Tinytest.add('Database - Redis Oplog - Removed', function (test) { const pub = RegisterPublication(() => TestData.find({})); From a3ae6baa2f29b5e1a197800ae6f8d08ac4534a4e Mon Sep 17 00:00:00 2001 From: Renan Date: Thu, 28 Dec 2023 16:01:33 -0300 Subject: [PATCH 44/50] fix: tests run without redis-oplog so we have to check it when initializing globals --- tests/_helpers/globals.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/_helpers/globals.js b/tests/_helpers/globals.js index 34fa13ca..bbfc0730 100644 --- a/tests/_helpers/globals.js +++ b/tests/_helpers/globals.js @@ -3,6 +3,6 @@ import { Meteor } from 'meteor/meteor'; export const MethodStore = []; export const TestData = new Meteor.Collection('tinytest-data'); export const TestDataRedis = new Meteor.Collection('tinytest-data-redis'); -TestDataRedis.configureRedisOplog({}); +TestDataRedis.configureRedisOplog?.({}); export const TestDataRedisNoRaceProtection = new Meteor.Collection('tinytest-data-redis-NoRaceProtection'); -TestDataRedisNoRaceProtection.configureRedisOplog({protectAgainstRaceConditions: false}); +TestDataRedisNoRaceProtection.configureRedisOplog?.({protectAgainstRaceConditions: false}); From ec30c6413c0ce48ee86b76669ab3f22accc4152a Mon Sep 17 00:00:00 2001 From: Renan Date: Thu, 28 Dec 2023 17:27:30 -0300 Subject: [PATCH 45/50] feat: add check for trace --- tests/hijack/redis_oplog.js | 46 ++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/tests/hijack/redis_oplog.js b/tests/hijack/redis_oplog.js index 56e16547..e1704f86 100644 --- a/tests/hijack/redis_oplog.js +++ b/tests/hijack/redis_oplog.js @@ -1,5 +1,5 @@ import { TestData, TestDataRedis, TestDataRedisNoRaceProtection } from '../_helpers/globals'; -import { GetMeteorClient, RegisterPublication, SubscribeAndWait } from '../_helpers/helpers'; +import { GetMeteorClient, RegisterMethod, RegisterPublication, SubscribeAndWait } from '../_helpers/helpers'; /** * We only track the observers coming from subscriptions (which have `ownerInfo`) @@ -76,8 +76,27 @@ Tinytest.add('Database - Redis Oplog - Added with limit/skip', function (test) { Meteor._sleepForMs(100); }); +Tinytest.add('Database - Redis Oplog - With protect against race condition - Check Trace', function (test) { + // in this case, the mutator will refetch the doc when publishing it + const methodId = RegisterMethod(() => TestDataRedis.update({name: 'test'}, {$set: {name: 'abv'}})); -Tinytest.add('Database - Redis Oplog - With protect against race condition', function (test) { + TestDataRedis.remove({}); + + TestDataRedis.insert({ name: 'test' }); + + const client = GetMeteorClient(); + client.call(methodId); + Meteor._sleepForMs(1000); + + let trace = Kadira.models.methods.tracerStore.currentMaxTrace[`method::${methodId}`]; + const dbEvents = trace.events.filter((o) => o[0] === 'db'); + test.equal(dbEvents.length, 2); + + TestDataRedis.remove({}); + + Meteor._sleepForMs(100); +}); +Tinytest.add('Database - Redis Oplog - With protect against race condition - check for finds after receiving the msg', function (test) { // in this case, every subscriber will refetch the doc once when receiving it const pub = RegisterPublication(() => TestDataRedis.find({name: 'test'})); @@ -117,7 +136,7 @@ Tinytest.add('Database - Redis Oplog - With protect against race condition', fun Meteor._sleepForMs(100); }); -Tinytest.add('Database - Redis Oplog - Without protect against race condition', function (test) { +Tinytest.add('Database - Redis Oplog - Without protect against race condition - no extraneous finds', function (test) { // in this case, no subscriber will refetch the doc when receiving it const pub = RegisterPublication(() => TestDataRedisNoRaceProtection.find({})); @@ -155,6 +174,27 @@ Tinytest.add('Database - Redis Oplog - Without protect against race condition', Meteor._sleepForMs(100); }); +Tinytest.add('Database - Redis Oplog - Without protect against race condition - Check Trace', function (test) { + // in this case, the mutator will refetch the doc when publishing it + const methodId = RegisterMethod(() => TestDataRedisNoRaceProtection.update({name: 'test'}, {$set: {name: 'abv'}})); + + TestDataRedisNoRaceProtection.remove({}); + + TestDataRedisNoRaceProtection.insert({ name: 'test' }); + + const client = GetMeteorClient(); + client.call(methodId); + Meteor._sleepForMs(1000); + + let trace = Kadira.models.methods.tracerStore.currentMaxTrace[`method::${methodId}`]; + const dbEvents = trace.events.filter((o) => o[0] === 'db'); + test.equal(dbEvents.length, 3); + test.equal(dbEvents[2][2].func, 'fetch'); + + TestDataRedisNoRaceProtection.remove({}); + + Meteor._sleepForMs(100); +}); Tinytest.add('Database - Redis Oplog - Removed', function (test) { const pub = RegisterPublication(() => TestData.find({})); From 47a6a6ef398c1515a2ae1be14d9256fb279b0cda Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 6 Mar 2024 21:30:51 -0300 Subject: [PATCH 46/50] fix: merge conflict --- tests/_helpers/helpers.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/_helpers/helpers.js b/tests/_helpers/helpers.js index b5ee7bfd..1437766e 100644 --- a/tests/_helpers/helpers.js +++ b/tests/_helpers/helpers.js @@ -5,8 +5,6 @@ import { MethodStore, TestData } from './globals'; const Future = Npm.require('fibers/future'); -const Future = Npm.require('fibers/future'); - export const GetMeteorClient = function (_url) { const url = _url || Meteor.absoluteUrl(); return DDP.connect(url, {retry: false, }); From 324eff94f047723316381618aa578747fa7e4303 Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 6 Mar 2024 21:35:42 -0300 Subject: [PATCH 47/50] fix: test on older versions --- tests/_helpers/globals.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_helpers/globals.js b/tests/_helpers/globals.js index bbfc0730..96ecd27d 100644 --- a/tests/_helpers/globals.js +++ b/tests/_helpers/globals.js @@ -3,6 +3,6 @@ import { Meteor } from 'meteor/meteor'; export const MethodStore = []; export const TestData = new Meteor.Collection('tinytest-data'); export const TestDataRedis = new Meteor.Collection('tinytest-data-redis'); -TestDataRedis.configureRedisOplog?.({}); +if (TestDataRedis.configureRedisOplog) TestDataRedis.configureRedisOplog({}); export const TestDataRedisNoRaceProtection = new Meteor.Collection('tinytest-data-redis-NoRaceProtection'); TestDataRedisNoRaceProtection.configureRedisOplog?.({protectAgainstRaceConditions: false}); From 778e3e79010b19402b6ab235a9a248df4054bd07 Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 6 Mar 2024 21:39:52 -0300 Subject: [PATCH 48/50] fix: test on older versions --- tests/_helpers/globals.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_helpers/globals.js b/tests/_helpers/globals.js index 96ecd27d..3ae84ac6 100644 --- a/tests/_helpers/globals.js +++ b/tests/_helpers/globals.js @@ -5,4 +5,4 @@ export const TestData = new Meteor.Collection('tinytest-data'); export const TestDataRedis = new Meteor.Collection('tinytest-data-redis'); if (TestDataRedis.configureRedisOplog) TestDataRedis.configureRedisOplog({}); export const TestDataRedisNoRaceProtection = new Meteor.Collection('tinytest-data-redis-NoRaceProtection'); -TestDataRedisNoRaceProtection.configureRedisOplog?.({protectAgainstRaceConditions: false}); +if (TestDataRedisNoRaceProtection.configureRedisOplog) TestDataRedisNoRaceProtection.configureRedisOplog({protectAgainstRaceConditions: false}); From 257d5d704048408013e2776088d5b2ede68860d6 Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 3 Apr 2024 19:42:05 -0300 Subject: [PATCH 49/50] fix: include all versions on final testing --- .github/workflows/test-redis-oplog.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml index 833cf010..d9d6b6c1 100644 --- a/.github/workflows/test-redis-oplog.yml +++ b/.github/workflows/test-redis-oplog.yml @@ -12,9 +12,26 @@ jobs: fail-fast: false matrix: meteorRelease: + - '--release 1.4.4.6' + - '--release 1.5.4.1' + - '--release 1.6.0.1' + - '--release 1.7.0.5' + - '--release 1.8.1' + - '--release 1.8.3' + - '--release 1.9.1' + - '--release 1.10.2' + - '--release 1.11' - '--release 1.12.1' - '--release 2.1.1' - # Latest version + - '--release 2.2' + - '--release 2.3.2' + - '--release 2.4.1' + - '--release 2.5.6' + - '--release 2.6.1' + - '--release 2.7.3' + - '--release 2.8.2' + - '--release 2.9.0' + # Left empty to use latest version - env: REDIS_OPLOG_SETTINGS: '{"debug":true}' From e12f3dc26fff7160cff8d082017e64a8a35011fa Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 3 Apr 2024 19:49:58 -0300 Subject: [PATCH 50/50] fix: remove some versions on final testing --- .github/workflows/test-redis-oplog.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test-redis-oplog.yml b/.github/workflows/test-redis-oplog.yml index d9d6b6c1..2b7dddac 100644 --- a/.github/workflows/test-redis-oplog.yml +++ b/.github/workflows/test-redis-oplog.yml @@ -12,14 +12,6 @@ jobs: fail-fast: false matrix: meteorRelease: - - '--release 1.4.4.6' - - '--release 1.5.4.1' - - '--release 1.6.0.1' - - '--release 1.7.0.5' - - '--release 1.8.1' - - '--release 1.8.3' - - '--release 1.9.1' - - '--release 1.10.2' - '--release 1.11' - '--release 1.12.1' - '--release 2.1.1'