Merge pull request #349 from dadi/release/3.0.0

API Version 3.0.0
dadi · Oct 31, 2017 · 02db906 · 02db906
2 parents a8e8b3c + f5e37da
commit 02db906
Show file tree

Hide file tree

Showing 91 changed files with 9,556 additions and 7,344 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,11 +3,9 @@ cache:
   directories:
     - node_modules
 notifications:
-  slack: dadi:pnhiL60xOrm7GOglHUmb7xHK
   email: false
 node_js:
   - '6'
-  - '5'
 before_install:
   - if [[ `npm -v` != 3* ]]; then npm i -g npm@latest; fi
 before_script:

diff --git a/AUTHORS b/AUTHORS
@@ -0,0 +1,14 @@
+Adam K Dean <[email protected]>
+Arthur Mingard <[email protected]>
+Dave Allen <[email protected]>
+David Longworth <[email protected]>
+Derrick <[email protected]>
+Eduardo Boucas <[email protected]>
+Francesco Iannuzzelli <[email protected]>
+James Lambie <[email protected]>
+Joe Wagner
+Joseph Denne <[email protected]>
+Kevin Sowers <[email protected]>
+Magnus Dahlstrand <[email protected]>
+Robert Stanford <[email protected]>
+Viktor Fero <[email protected]>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,69 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [3.0.0] (2017-11-02)
+
+See full release notes at https://github.com/dadi/api/releases/tag/v3.0.0
+
+### Added
+
+#### Data Connectors
+
+API Version 3.0 supports multiple data connectors. In previous versions API used MongoDB as a backend; this is now configurable. API Data Connectors are available as NPM packages. To add one to your API installation, run the associated NPM install command:
+
+`$ npm install @dadi/api-mongodb --save`
+
+Each data connector has it's own configuration requirements, but API must also be configured to use the data connectors you select. Modify your API configuration as follows:
+
+```json
+{
+  "datastore": "@dadi/api-mongodb",  // the NPM package name for the data connector to use for the content layer
+  "auth": {
+    "tokenUrl": "/token",
+    "tokenTtl": 1800,
+    "clientCollection": "clientStore",
+    "tokenCollection": "tokenStore",
+    "datastore": "@dadi/api-mongodb",  // the NPM package name for the data connector to use for the authentication layer
+    "database": "test"
+  }
+}
+```
+
+In addition, the data connector itself normally requires it's own configuration file. For example the MongoDB data connector requires a file using the following naming convention `mongodb.<environment>.json`. These configuration files should be placed the `config` directory of the API application.
+
+#### Connection Recovery
+
+API is now capable of recovering from database connection failures. When API is started with no available database service it will keep retrying until a successful connection can be made, then it runs the normal boot process.
+
+In addition, if the database connection is lost during normal operation of API, any requests made while the connection is offline will result in a HTTP 503 returned to the client.
+
+The maximum number of connection retries can be configured in the main configuration file by adding the following block:
+
+```json
+"databaseConnection": {
+  "maxRetries": 5   // default 10
+}
+```
+
+### Changed
+
+* [#141](https://github.com/dadi/api/issues/141): the internal fields will be prefixed with a special character (`_` by default) which is configurable using the configuration property `internalFieldsPrefix`
+* [#180](https://github.com/dadi/api/issues/180): document properties with `null` values are not returned as part of the response
+* [#251](https://github.com/dadi/api/issues/251): added a new `/hello` endpoint which returns HTTP 200 and a "Welcome to API" message
+* [#263](https://github.com/dadi/api/issues/263): all delete hooks now receive a `deletedDocs` property
+* [#314](https://github.com/dadi/api/issues/314): when configuration option `feedback` is `true` we now send a response body when deleting documents
+* [#327](https://github.com/dadi/api/issues/327): API becomes capable of recovering from database connection failures
+* [#328](https://github.com/dadi/api/issues/328): remove schema validation on settings: 'callback', 'defaultFilters', 'fieldLimiters' and 'count'. Now only requires 'cache' and 'authenticate'
+* [#332](https://github.com/dadi/api/issues/332): allow POST to collection endpoints using `text/plain` content-type, which will be converted if it is valid JSON
+* Configuration file validation removed, suppressing warnings on application startup
+* POST/PUT/DELETE using non-existing document identifiers returns a 404:
+
+DELETE requests throws a 404 (instead of 204) when deleting a non-existing document by ID. This applies to requests where the document ID is passed in the URL, not when in the body (e.g. DELETE /v1/db/collection/DOC-ID vs DELETE /v1/db/collection).
+
+POST/PUT requests throw a 404 when updating a non-existing document by ID. This applies to requests where the document ID is passed in the URL, not when in the body (e.g. PUT /v1/db/collection/DOC-ID vs PUT /v1/db/collection).
+
+Closes [#345](https://github.com/dadi/api/issues/345).
+
 ## [2.2.0] (2017-07-05)
 
 ### Changed

diff --git a/README.md b/README.md
@@ -159,6 +159,25 @@ $ [sudo] service api restart
 ## Links
 * [API Documentation](http://docs.dadi.tech/api/)
 
+## Contributors
+
+DADI API is based on an original idea by Joseph Denne. It is developed and maintained by the engineering team at DADI ([https://dadi.tech](https://dadi.tech))
+
+* Adam K Dean <[email protected]>
+* Arthur Mingard <[email protected]>
+* Dave Allen <[email protected]>
+* David Longworth <[email protected]>
+* Derrick <[email protected]>
+* Eduardo Boucas <[email protected]>
+* Francesco Iannuzzelli <[email protected]>
+* James Lambie <[email protected]>
+* Joe Wagner
+* Joseph Denne <[email protected]>
+* Kevin Sowers <[email protected]>
+* Magnus Dahlstrand <[email protected]>
+* Robert Stanford <[email protected]>
+* Viktor Fero <[email protected]>
+
 ## Licence
 
 DADI is a data centric development and delivery stack, built specifically in support of the principles of API first and COPE.
@@ -193,3 +212,4 @@ A copy can be found in the file GPL.md distributed with
 these files.
 
 This copyright notice MUST APPEAR in all copies of the product!
+
diff --git a/config.js b/config.js
@@ -1,7 +1,6 @@
 var convict = require('convict')
 var fs = require('fs')
 
-// Define a schema
 var conf = convict({
   app: {
     name: {
@@ -91,55 +90,10 @@ var conf = convict({
       env: 'SSL_INTERMEDIATE_CERTIFICATE_PATHS'
     }
   },
-  database: {
-      hosts: {
-        doc: '',
-        format: Array,
-        default: [
-          {
-            host: '127.0.0.1',
-            port: 27017
-          }
-        ]
-      },
-      username: {
-        doc: '',
-        format: String,
-        default: '',
-        env: 'DB_USERNAME'
-      },
-      password: {
-        doc: '',
-        format: String,
-        default: '',
-        env: 'DB_PASSWORD'
-      },
-      database: {
-        doc: '',
-        format: String,
-        default: 'test',
-        env: 'DB_NAME'
-      },
-      ssl: {
-        doc: '',
-        format: Boolean,
-        default: false
-      },
-      replicaSet: {
-        doc: '',
-        format: String,
-        default: ''
-    },
-    readPreference: {
-      doc: "Choose how MongoDB routes read operations to the members of a replica set - see https://docs.mongodb.com/manual/reference/read-preference/",
-      format: ['primary', 'primaryPreferred', 'secondary', 'secondaryPreferred', 'nearest'],
-      default: 'secondaryPreferred'
-      },
-      enableCollectionDatabases: {
-        doc: '',
-        format: Boolean,
-        default: false
-      }
+  datastore: {
+    doc: "",
+    format: String,
+    default: '@dadi/api-mongodb'
   },
   auth: {
     tokenUrl: {
@@ -160,37 +114,23 @@ var conf = convict({
     tokenCollection: {
       doc: '',
       format: String,
-      default: 'tokenStore'
+      default: "tokenStore"
+    },
+    datastore: {
+      doc: "",
+      format: String,
+      default: '@dadi/api-mongodb'
     },
     database: {
-      hosts: {
-        doc: '',
-        format: Array,
-        default: [
-          {
-            host: '127.0.0.1',
-            port: 27017
-          }
-        ]
-      },
-      username: {
-        doc: '',
-        format: String,
-        default: '',
-        env: 'DB_AUTH_USERNAME'
-      },
-      password: {
-        doc: '',
-        format: String,
-        default: '',
-        env: 'DB_AUTH_PASSWORD'
-      },
-      database: {
-        doc: '',
-        format: String,
-        default: 'test',
-        env: 'DB_AUTH_NAME'
-      }
+      doc: '',
+      format: String,
+      default: 'test',
+      env: 'DB_AUTH_NAME'
+    },
+    cleanupInterval: {
+      doc: 'The interval (in seconds) at which the token store will delete expired tokens from the database',
+      format: Number,
+      default: 3600
     }
   },
   caching: {
@@ -214,6 +154,16 @@ var conf = convict({
         doc: 'The extension to use for cache files',
         format: String,
         default: 'json'
+      },
+      autoFlush: {
+        doc: "",
+        format: Boolean,
+        default: true
+      },
+      autoFlushInterval: {
+        doc: "",
+        format: Number,
+        default: 60
       }
     },
     redis: {
@@ -284,12 +234,20 @@ var conf = convict({
     }
   },
   paths: {
-    doc: '',
-    format: Object,
-    default: {
-      collections: __dirname + '/workspace/collections',
-      endpoints: __dirname + '/workspace/endpoints',
-      hooks: __dirname + '/workspace/hooks'
+    collections: {
+      doc: 'The relative or absolute path to collection specification files',
+      format: String,
+      default: __dirname + '/workspace/collections'
+    },
+    endpoints: {
+      doc: 'The relative or absolute path to custom endpoint files',
+      format: String,
+      default: __dirname + '/workspace/endpoints'
+    },
+    hooks: {
+      doc: 'The relative or absolute path to hook specification files',
+      format: String,
+      default: __dirname + '/workspace/hooks'
     }
   },
   feedback: {
@@ -395,24 +353,33 @@ var conf = convict({
     doc: 'If true, responses will include headers for cross-domain resource sharing',
     format: Boolean,
     default: false
+  },
+  internalFieldsPrefix: {
+    doc: 'The character to be used for prefixing internal fields',
+    format: 'String',
+    default: '_'
+  },
+  databaseConnection: {
+    maxRetries: {
+      doc: "The maximum number of times to reconnection attempts after a database fails",
+      format: Number,
+      default: 10
+    }
   }
 })
 
 // Load environment dependent configuration
 var env = conf.get('env')
 conf.loadFile('./config/config.' + env + '.json')
 
-// Perform validation
-conf.validate()
-
 // Load domain-specific configuration
 conf.updateConfigDataForDomain = function(domain) {
   var domainConfig = './config/' + domain + '.json'
+
   try {
     var stats = fs.statSync(domainConfig)
     // no error, file exists
     conf.loadFile(domainConfig)
-    conf.validate()
   }
   catch(err) {
     if (err.code === 'ENOENT') {

diff --git a/config/config.development.json.sample b/config/config.development.json.sample
@@ -10,58 +10,14 @@
     "host": "localhost",
     "port": 3000
   },
-  "database": {
-    "hosts": [
-      {
-        "host": "127.0.0.1",
-        "port": 27017
-      }
-    ],
-    "username": "",
-    "password": "",
-    "database": "api",
-    "ssl": false,
-    "replicaSet": "",
-    "enableCollectionDatabases": false,
-    "secondary": {
-      "hosts": [
-        {
-          "host": "127.0.0.1",
-          "port": 27018
-        }
-      ],
-      "username": "",
-      "password": "",
-      "replicaSet": "",
-      "ssl": false
-    },
-    "testdb": {
-      "hosts": [
-        {
-          "host": "127.0.0.1",
-          "port": 27017
-        }
-      ],
-      "username": "",
-      "password": ""
-    }
-  },
+  "datastore": "@dadi/api-mongodb",
   "auth": {
     "tokenUrl": "/token",
-    "tokenTtl": 1800,
-    "clientCollection": "clientStore",
-    "tokenCollection": "tokenStore",
-    "database": {
-      "hosts": [
-        {
-          "host": "127.0.0.1",
-          "port": 27017
-        }
-      ],
-      "username": "",
-      "password": "",
-      "database": "api"
-    }
+    "tokenTtl": 1800000,
+    "clientCollection": "client-store",
+    "tokenCollection": "token-store",
+    "datastore": "@dadi/api-mongodb",
+    "database": "testdb"
   },
   "caching": {
     "ttl": 300,