Feature/exclusions

.gitignore

@@ -1,3 +1,5 @@
 node_modules
 tmp*
-coverage*
+coverage*
+/.idea/**
+/backups/**

README.md

@@ -53,6 +53,7 @@ Options:
 * `-B`, `--backupPath` `<path>` Path to store backup.
 * `-a2`, `--restoreAccountCredentials` `<path>` Google Cloud account credentials JSON file for restoring documents.
 * `-P`, `--prettyPrint` JSON backups done with pretty-printing.
+* `-E`, `--exclude` Comma separated list of excluded collections, use * for document id wildcard.
 * `-S`, `--stable` JSON backups done with stable-stringify.
 * `-J`, `--plainJSONBackup` JSON backups done without preserving any type information. - Lacks full fidelity restore to Firestore. - Can be used for other export purposes.
 * `-h`, `--help` output usage information
@@ -103,6 +104,21 @@ Example:
 firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --prettyPrint
 ```
 
+### Exclude collection or document
+
+To exclude collections or documents from backup you can use the option `--exclude`.
+
+* `-E`, `--exclude` - Comma separated list of excluded paths, use * for document id wildcard.
+
+All documents and subcollections following a match will be excluded. By specifying the document ID a specific subcollection can be excluded. Use `*` for document ID wildcard.    
+
+Example:
+
+```sh
+firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase \
+  --exclude "/logs,/users/*/details,/settings/private"
+```
+
 ### Backup with stable stringify:
 
 If you want the json documents to have sorted keys, then use the `--stable` option.

build/index.js

@@ -49,6 +49,9 @@ var accountCredentialsPathParamDescription = 'Google Cloud account credentials J
 var backupPathParamKey = 'backupPath';
 var backupPathParamDescription = 'Path to store backup';
 
+var excludeCollectionsParamKey = 'exclude';
+var excludeCollectionsParamDescription = 'Comma separated list of excluded collections, ' + 'use * for document id wildcard';
+
 var restoreAccountCredentialsPathParamKey = 'restoreAccountCredentials';
 var restoreAccountCredentialsPathParamDescription = 'Google Cloud account credentials JSON file for restoring documents';
 
@@ -72,7 +75,7 @@ try {
 // or they can be merged with existing ones
 var mergeData = false;
 
-_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).parse(_process2.default.argv);
+_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-E, --' + excludeCollectionsParamKey + ' <col1,col2>', excludeCollectionsParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).parse(_process2.default.argv);
 
 var accountCredentialsPath = _commander2.default[accountCredentialsPathParamKey];
 if (accountCredentialsPath && !_fs2.default.existsSync(accountCredentialsPath)) {
@@ -95,6 +98,8 @@ if (restoreAccountCredentialsPath && !_fs2.default.existsSync(restoreAccountCred
   _process2.default.exit(1);
 }
 
+var exclusions = _commander2.default[excludeCollectionsParamKey] !== undefined && _commander2.default[excludeCollectionsParamKey] !== null ? _commander2.default[excludeCollectionsParamKey].split(',') : [];
+
 var prettyPrint = _commander2.default[prettyPrintParamKey] !== undefined && _commander2.default[prettyPrintParamKey] !== null;
 
 var stable = _commander2.default[stableParamKey] !== undefined && _commander2.default[stableParamKey] !== null;
@@ -123,8 +128,32 @@ var promiseSerial = function promiseSerial(funcs) {
   }, Promise.resolve([]));
 };
 
+var isExcluded = function isExcluded(fullPath) {
+  if (exclusions.length > 0) {
+    var escapedPath = fullPath.split('/').reduce(function (previousValue, currentValue, currentIndex) {
+      return previousValue + (currentIndex % 2 === 1 ? '/' + currentValue : '/*');
+    });
+    return exclusions.find(function (exclusion) {
+      return fullPath.startsWith(exclusion);
+    }) || exclusions.find(function (exclusion) {
+      return escapedPath.startsWith(exclusion);
+    });
+  }
+  return false;
+};
+
 var backupDocument = function backupDocument(document, backupPath, logPath) {
-  console.log("Backing up Document '" + logPath + document.id + "'" + (plainJSONBackup === true ? ' with -J --plainJSONBackup' : ' with type information'));
+
+  var fullPath = logPath + document.id;
+
+  if (isExcluded(fullPath)) {
+    console.log('Skipping backup of Document ' + fullPath);
+    return promiseSerial([function () {
+      return Promise.resolve();
+    }]);
+  }
+
+  console.log("Backing up Document '" + fullPath + "'" + (plainJSONBackup === true ? ' with -J --plainJSONBackup' : ' with type information'));
 
   try {
     _mkdirp2.default.sync(backupPath);
@@ -148,7 +177,7 @@ var backupDocument = function backupDocument(document, backupPath, logPath) {
     return document.ref.getCollections().then(function (collections) {
       return promiseSerial(collections.map(function (collection) {
         return function () {
-          return backupCollection(collection, backupPath + '/' + collection.id, logPath + document.id + '/');
+          return backupCollection(collection, backupPath + '/' + collection.id, fullPath + '/');
         };
       }));
     });
@@ -159,13 +188,29 @@ var backupDocument = function backupDocument(document, backupPath, logPath) {
 };
 
 var backupCollection = function backupCollection(collection, backupPath, logPath) {
-  console.log("Backing up Collection '" + logPath + collection.id + "'");
+  var fullPath = logPath + collection.id;
+
+  if (isExcluded(fullPath)) {
+    console.log('Skipping backup of Collection ' + fullPath);
+    return promiseSerial([function () {
+      return Promise.resolve();
+    }]);
+  }
 
-  // TODO: implement feature to skip certain Collections
-  // if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
-  //   console.log(`Skipping ${collection.id}`);
-  //   return promiseSerial([() => Promise.resolve()]);
-  // }
+  if (exclusions.length > 0) {
+    var escapedPath = fullPath.split('/').reduce(function (previousValue, currentValue, currentIndex) {
+      return previousValue + (currentIndex % 2 === 1 ? '/' + currentValue : '/*');
+    });
+    if (exclusions.find(function (exclusion) {
+      return fullPath.startsWith(exclusion);
+    }) || exclusions.find(function (exclusion) {
+      return escapedPath.startsWith(exclusion);
+    })) {
+      return promiseSerial([function () {
+        return Promise.resolve();
+      }]);
+    }
+  }
 
   try {
     _mkdirp2.default.sync(backupPath);
@@ -174,8 +219,8 @@ var backupCollection = function backupCollection(collection, backupPath, logPath
       var backupFunctions = [];
       snapshots.forEach(function (document) {
         backupFunctions.push(function () {
-          var backupDocumentPromise = backupDocument(document, backupPath + '/' + document.id, logPath + collection.id + '/');
-          restoreDocument(logPath + collection.id, document);
+          var backupDocumentPromise = backupDocument(document, backupPath + '/' + document.id, fullPath + '/');
+          restoreDocument(fullPath, document);
           return backupDocumentPromise;
         });
       });

index.js

@@ -24,6 +24,10 @@ const accountCredentialsPathParamDescription =
 const backupPathParamKey = 'backupPath';
 const backupPathParamDescription = 'Path to store backup';
 
+const excludeCollectionsParamKey = 'exclude'
+const excludeCollectionsParamDescription = 'Comma separated list of excluded collections, ' +
+  'use * for document id wildcard';
+
 const restoreAccountCredentialsPathParamKey = 'restoreAccountCredentials';
 const restoreAccountCredentialsPathParamDescription =
   'Google Cloud account credentials JSON file for restoring documents';
@@ -61,6 +65,7 @@ commander
     '-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>',
     restoreAccountCredentialsPathParamDescription
   )
+  .option('-E, --' + excludeCollectionsParamKey + ' <col1,col2>', excludeCollectionsParamDescription)
   .option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription)
   .option('-S, --' + stableParamKey, stableParamParamDescription)
 
@@ -104,6 +109,12 @@ if (
   process.exit(1);
 }
 
+const exclusions =
+  (commander[excludeCollectionsParamKey] !== undefined &&
+    commander[excludeCollectionsParamKey] !== null)
+    ? commander[excludeCollectionsParamKey].split(',')
+    : [];
+
 const prettyPrint =
   commander[prettyPrintParamKey] !== undefined &&
   commander[prettyPrintParamKey] !== null;
@@ -147,15 +158,33 @@ const promiseSerial = funcs => {
   }, Promise.resolve([]));
 };
 
+const isExcluded = fullPath => {
+  if (exclusions.length > 0) {
+    const escapedPath = fullPath.split('/').reduce((previousValue, currentValue, currentIndex) => {
+      return previousValue + (currentIndex % 2 === 1 ? `/${currentValue}` : '/*');
+    })
+    return exclusions.find(exclusion => fullPath.startsWith(exclusion)) ||
+      exclusions.find(exclusion => escapedPath.startsWith(exclusion));
+  }
+  return false;
+};
+
 const backupDocument = (
   document: Object,
   backupPath: string,
   logPath: string
 ): Promise<any> => {
+
+  const fullPath = logPath + document.id
+
+  if (isExcluded(fullPath)) {
+    console.log(`Skipping backup of Document ${fullPath}`);
+    return promiseSerial([() => Promise.resolve()]);
+  }
+
   console.log(
     "Backing up Document '" +
-      logPath +
-      document.id +
+      fullPath +
       "'" +
       (plainJSONBackup === true
         ? ' with -J --plainJSONBackup'
@@ -191,7 +220,7 @@ const backupDocument = (
             return backupCollection(
               collection,
               backupPath + '/' + collection.id,
-              logPath + document.id + '/'
+              fullPath + '/'
             );
           };
         })
@@ -219,13 +248,22 @@ const backupCollection = (
   backupPath: string,
   logPath: string
 ): Promise<void> => {
-  console.log("Backing up Collection '" + logPath + collection.id + "'");
+  const fullPath = logPath + collection.id
+
+  if (isExcluded(fullPath)) {
+    console.log(`Skipping backup of Collection ${fullPath}`);
+    return promiseSerial([() => Promise.resolve()]);
+  }
 
-  // TODO: implement feature to skip certain Collections
-  // if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
-  //   console.log(`Skipping ${collection.id}`);
-  //   return promiseSerial([() => Promise.resolve()]);
-  // }
+  if (exclusions.length > 0) {
+    const escapedPath = fullPath.split('/').reduce((previousValue, currentValue, currentIndex) => {
+      return previousValue + (currentIndex % 2 === 1 ? `/${currentValue}` : '/*')
+    })
+    if (exclusions.find(exclusion => fullPath.startsWith(exclusion)) ||
+      exclusions.find(exclusion => escapedPath.startsWith(exclusion))) {
+      return promiseSerial([() => Promise.resolve()])
+    }
+  }
 
   try {
     mkdirp.sync(backupPath);
@@ -237,9 +275,9 @@ const backupCollection = (
           const backupDocumentPromise = backupDocument(
             document,
             backupPath + '/' + document.id,
-            logPath + collection.id + '/'
+            fullPath + '/'
           );
-          restoreDocument(logPath + collection.id, document);
+          restoreDocument(fullPath, document);
           return backupDocumentPromise;
         });
       });

package.json

@@ -26,18 +26,15 @@
     "prepublishOnly": "npm run clean && npm run build"
   },
   "bin": "./bin/firestore-backup-restore.js",
-  "dependenciesComments": {
-    "@google-cloud/firestore": "FIXME - Can be removed once next version after 0.11.2 is released which exposes all the Firestore types so instanceof can be used"
-  },
   "dependencies": {
-    "@google-cloud/firestore": "googleapis/nodejs-firestore",
+    "@google-cloud/firestore": "^0.19.0",
     "babel-cli": "^6.26.0",
-    "babel-preset-env": "^1.6.1",
+    "babel-preset-env": "^1.7.0",
     "babel-preset-flow": "^6.23.0",
     "babel-runtime": "^6.26.0",
-    "colors": "^1.1.2",
-    "commander": "^2.11.0",
-    "firebase-admin": "^5.4.3",
+    "colors": "^1.3.2",
+    "commander": "^2.19.0",
+    "firebase-admin": "^6.2.0",
     "json-stable-stringify": "^1.0.1",
     "mkdirp": "^0.5.1"
   },
@@ -46,10 +43,10 @@
     "babel-plugin-transform-object-rest-spread": "^6.26.0",
     "babel-plugin-transform-runtime": "^6.23.0",
     "eslint": "3.x",
-    "eslint-plugin-flowtype": "^2.39.1",
-    "flow-bin": "^0.64.0",
-    "jest": "^22.1.4",
-    "standard": "^10.0.3"
+    "eslint-plugin-flowtype": "^3.2.0",
+    "flow-bin": "^0.86.0",
+    "jest": "^23.6.0",
+    "standard": "^12.0.1"
   },
   "jest": {
     "collectCoverageFrom": [