Merge pull request #1 from jzlai/exclude-collections

Exclude collections

README.md

@@ -56,6 +56,8 @@ Options:
 * `-S`, `--stable` JSON backups done with stable-stringify.
 * `-J`, `--plainJSONBackup` JSON backups done without preserving any type information. - Lacks full fidelity restore to Firestore. - Can be used for other export purposes.
 * `-h`, `--help` output usage information
+* `-e`, `--excludeCollections` Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[/collection3]
+* `-E`, `--excludePattern` Exclude patterns to match against when backing up, e.g. [regex1],[regex2]
 
 ### Backup:
 
@@ -202,6 +204,25 @@ Document saved with type information (Default)
   identifier: { value: 'provider', type: 'string' }
 ```
 
+### Exclude collections
+
+The optional parameter allows `--excludeCollections` skipping of provided collections. This parameter accepts a comma seperated list of collections.
+
+Example:
+
+```sh
+firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludeCollections /collection1/document/subcollectionToIgnore,/collectionToIgnore
+```
+
+### Exclude paths by regex
+Skips documents or collections by pattern matching. All subpaths of matched paths will also be excluded. This parameter accepts a comma seperated list of regular expressions.
+
+Example:
+
+```sh
+firestore-backup-restore --accountCredentials path/to/account/credentials/file.json --backupPath /backups/myDatabase --excludePattern '^/collectionToIgnore,^/[^/]*/[^/]*/subcollectionToIgnore'
+```
+
 ## Contributions
 
 Feel free to report bugs in the [Issue Tracker](https://github.com/willhlaw/node-firestore-backup-restore/issues), fork and create pull requests!

build/index.js

@@ -43,6 +43,16 @@ var _FirestoreDocument = require('./lib/FirestoreDocument');
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+function commaSeparatedList(value) {
+  return value.split(',');
+}
+
+function commaSeparatedListAndRegExp(value) {
+  return value.split(',').map(function (entry) {
+    return new RegExp(entry);
+  });
+}
+
 var accountCredentialsPathParamKey = 'accountCredentials';
 var accountCredentialsPathParamDescription = 'Google Cloud account credentials JSON file';
 
@@ -60,6 +70,11 @@ var stableParamParamDescription = 'JSON backups done with stable-stringify';
 
 var plainJSONBackupParamKey = 'plainJSONBackup';
 var plainJSONBackupParamDescription = 'JSON backups done without preserving any type information\n                                          - Lacks full fidelity restore to Firestore\n                                          - Can be used for other export purposes';
+var excludeCollectionParamKey = 'excludeCollections';
+var excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]';
+
+var excludePatternParamKey = 'excludePattern';
+var excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]';
 
 var packagePath = __dirname.includes('/build') ? '..' : '.';
 
@@ -72,7 +87,7 @@ try {
 // or they can be merged with existing ones
 var mergeData = false;
 
-_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).parse(_process2.default.argv);
+_commander2.default.version(version).option('-a, --' + accountCredentialsPathParamKey + ' <path>', accountCredentialsPathParamDescription).option('-B, --' + backupPathParamKey + ' <path>', backupPathParamDescription).option('-a2, --' + restoreAccountCredentialsPathParamKey + ' <path>', restoreAccountCredentialsPathParamDescription).option('-P, --' + prettyPrintParamKey, prettyPrintParamDescription).option('-S, --' + stableParamKey, stableParamParamDescription).option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription).option('-e, --' + excludeCollectionParamKey + ' <collections>', excludeCollectionParamDescription, commaSeparatedList).option('-E, --' + excludePatternParamKey + ' <regex>', excludePatternParamDescription, commaSeparatedListAndRegExp).parse(_process2.default.argv);
 
 var accountCredentialsPath = _commander2.default[accountCredentialsPathParamKey];
 if (accountCredentialsPath && !_fs2.default.existsSync(accountCredentialsPath)) {
@@ -101,6 +116,10 @@ var stable = _commander2.default[stableParamKey] !== undefined && _commander2.de
 
 var plainJSONBackup = _commander2.default[plainJSONBackupParamKey] !== undefined && _commander2.default[plainJSONBackupParamKey] !== null;
 
+var excludeCollections = _commander2.default[excludeCollectionParamKey] || [];
+
+var excludePatterns = _commander2.default[excludePatternParamKey] || [];
+
 var accountApp = accountCredentialsPath ? (0, _FirestoreFunctions.getFireApp)(accountCredentialsPath) : {};
 
 try {
@@ -126,6 +145,15 @@ var promiseSerial = function promiseSerial(funcs) {
 var backupDocument = function backupDocument(document, backupPath, logPath) {
   console.log("Backing up Document '" + logPath + document.id + "'" + (plainJSONBackup === true ? ' with -J --plainJSONBackup' : ' with type information'));
 
+  if (excludePatterns.some(function (pattern) {
+    return pattern.test(logPath + document.id);
+  })) {
+    console.log('Skipping ' + document.id);
+    return promiseSerial([function () {
+      return Promise.resolve();
+    }]);
+  }
+
   try {
     _mkdirp2.default.sync(backupPath);
     var fileContents = void 0;
@@ -159,13 +187,17 @@ var backupDocument = function backupDocument(document, backupPath, logPath) {
 };
 
 var backupCollection = function backupCollection(collection, backupPath, logPath) {
-  console.log("Backing up Collection '" + logPath + collection.id + "'");
-
-  // TODO: implement feature to skip certain Collections
-  // if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
-  //   console.log(`Skipping ${collection.id}`);
-  //   return promiseSerial([() => Promise.resolve()]);
-  // }
+  var collectionPath = logPath + collection.id;
+  console.log("Backing up Collection '" + collectionPath + "'");
+
+  if (excludeCollections.includes(collectionPath) || excludePatterns.some(function (pattern) {
+    return pattern.test(collectionPath);
+  })) {
+    console.log('Skipping ' + collection.id);
+    return promiseSerial([function () {
+      return Promise.resolve();
+    }]);
+  }
 
   try {
     _mkdirp2.default.sync(backupPath);

index.js

@@ -17,6 +17,14 @@ import {
   saveDocument
 } from './lib/FirestoreDocument';
 
+function commaSeparatedList(value) {
+  return value.split(',')
+}
+
+function commaSeparatedListAndRegExp(value) {
+  return value.split(',').map(entry => new RegExp(entry))
+}
+
 const accountCredentialsPathParamKey = 'accountCredentials';
 const accountCredentialsPathParamDescription =
   'Google Cloud account credentials JSON file';
@@ -38,6 +46,11 @@ const plainJSONBackupParamKey = 'plainJSONBackup';
 const plainJSONBackupParamDescription = `JSON backups done without preserving any type information
                                           - Lacks full fidelity restore to Firestore
                                           - Can be used for other export purposes`;
+const excludeCollectionParamKey = 'excludeCollections';
+const excludeCollectionParamDescription = 'Excludes provided collections when backing up, e.g. [/collection1/doc1/subcollection2],[collection3]';
+
+const excludePatternParamKey = 'excludePattern';
+const excludePatternParamDescription = 'Exclude patterns to match against when backing up, e.g. [regex1],[regex2]'
 
 const packagePath = __dirname.includes('/build') ? '..' : '.';
 
@@ -65,6 +78,8 @@ commander
   .option('-S, --' + stableParamKey, stableParamParamDescription)
 
   .option('-J, --' + plainJSONBackupParamKey, plainJSONBackupParamDescription)
+  .option('-e, --' + excludeCollectionParamKey + ' <collections>', excludeCollectionParamDescription, commaSeparatedList)
+  .option('-E, --' + excludePatternParamKey + ' <regex>', excludePatternParamDescription, commaSeparatedListAndRegExp)
   .parse(process.argv);
 
 const accountCredentialsPath = commander[accountCredentialsPathParamKey];
@@ -116,6 +131,10 @@ const plainJSONBackup =
   commander[plainJSONBackupParamKey] !== undefined &&
   commander[plainJSONBackupParamKey] !== null;
 
+const excludeCollections = commander[excludeCollectionParamKey] || [];
+
+const excludePatterns = commander[excludePatternParamKey] || [];
+
 const accountApp: Object = accountCredentialsPath
   ? getFireApp(accountCredentialsPath)
   : {};
@@ -162,6 +181,11 @@ const backupDocument = (
         : ' with type information')
   );
 
+  if(excludePatterns.some(pattern => pattern.test(logPath + document.id))){
+    console.log(`Skipping ${document.id}`);
+    return promiseSerial([() => Promise.resolve()])
+  }
+
   try {
     mkdirp.sync(backupPath);
     let fileContents: string;
@@ -219,13 +243,13 @@ const backupCollection = (
   backupPath: string,
   logPath: string
 ): Promise<void> => {
-  console.log("Backing up Collection '" + logPath + collection.id + "'");
+  const collectionPath = logPath + collection.id
+  console.log("Backing up Collection '" + collectionPath + "'");
 
-  // TODO: implement feature to skip certain Collections
-  // if (collection.id.toLowerCase().indexOf('geotrack') > 0) {
-  //   console.log(`Skipping ${collection.id}`);
-  //   return promiseSerial([() => Promise.resolve()]);
-  // }
+  if (excludeCollections.includes(collectionPath) || excludePatterns.some(pattern => pattern.test(collectionPath))) {
+    console.log(`Skipping ${collection.id}`);
+    return promiseSerial([() => Promise.resolve()]);
+  }
 
   try {
     mkdirp.sync(backupPath);