Merge branch 'release-3.2.0'

.npmignore

@@ -0,0 +1,8 @@
+.github
+.log
+coverage/
+test/
+log/
+cache/
+config/*json
+!config/*json.sample

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [3.2.0] (2018-06-08)
+
+See release notes at https://github.com/dadi/api/releases/tag/v3.2.0
+
+### Added
+
+* [#431](https://github.com/dadi/api/pull/431): allow DELETE requests to media collection endpoints; enable `s3.endpoint` in the configuration to allow using Digital Ocean Spaces as a storage handler.
+
 ## [3.1.2] (2018-05-01)
 
 ### Changed

README.md

@@ -1,4 +1,4 @@
-<img src="https://dadi.tech/assets/products/dadi-api-full.png" alt="DADI API" height="65"/>
+<img src="https://dadi.cloud/assets/products/dadi-api-full.png" alt="DADI API" height="65"/>
 
 [![npm (scoped)](https://img.shields.io/npm/v/@dadi/api.svg?maxAge=10800&style=flat-square)](https://www.npmjs.com/package/@dadi/api)
 [![coverage](https://img.shields.io/badge/coverage-88%25-yellow.svg?style=flat)](https://github.com/dadi/api)
@@ -14,9 +14,9 @@
 
 ## Overview
 
-DADI API is built on Node.JS. It is a high performance RESTful API layer designed in support of [API-first development and the principle of COPE](https://dadi.tech/platform/concepts/api-first-and-cope/). It can use virtually any database engine, such as [MongoDB](https://github.com/dadi/api-mongodb), [CouchDB](https://github.com/dadi/api-couchdb), [RethinkDB](https://github.com/dadi/api-rethinkdb) or simply a [JSON filestore](https://github.com/dadi/api-filestore).
+DADI API is built on Node.JS. It is a high performance RESTful API layer designed in support of API-first development and the principle of COPE. It can use virtually any database engine, such as [MongoDB](https://github.com/dadi/api-mongodb), [CouchDB](https://github.com/dadi/api-couchdb), [RethinkDB](https://github.com/dadi/api-rethinkdb) or simply a [JSON filestore](https://github.com/dadi/api-filestore).
 
-You can consider it as the data layer within a platform (including the data model). It is designed to be plugged into a templating layer (such as [DADI Web](https://dadi.tech/platform/web)), a mobile application or to be used with any other data consumer.
+You can consider it as the data layer within a platform (including the data model). It is designed to be plugged into a templating layer (such as [DADI Web](https://dadi.cloud/en/web)), a mobile application or to be used with any other data consumer.
 
 Calls to a DADI API can contain your business/domain logic (the part of a platform that encodes the real-world business rules that determine how data is created, displayed, stored and changed). It has full support for searching, filtering, limiting, sorting, offsetting, input validation and data aggregation (through support for MongoDB's aggregation pipeline).
 
@@ -35,7 +35,7 @@ It is part of DADI, a suite of components covering the full development stack, b
 
 ### Install API
 
-The quickest way to get started with *API* is to use [DADI CLI](https://github.com/dadi/cli). See [Creating an API](https://docs.dadi.tech/#api/creating-an-api) for full installation details.
+The quickest way to get started with *API* is to use [DADI CLI](https://github.com/dadi/cli). See [Creating an API](https://docs.dadicloud/api) for full installation details.
 
 
 ### Configuration
@@ -95,7 +95,7 @@ Connection: keep-alive
 
 The HTTP 401 response received in the previous step shows that the server is running. To start using the REST endpoints you'll need a user account so you can obtain access tokens for interacting with the API.
 
-User accounts provide an authentication layer for API. Each user account has a *__clientId__* and a *__secret__*. These are used to obtain access tokens for interacting with the API. See the [Authentication](https://docs.dadi.tech/#api/authentication) section of the API documentation for full details.
+User accounts provide an authentication layer for API. Each user account has a *__clientId__* and a *__secret__*. These are used to obtain access tokens for interacting with the API. See the [Authentication](https://docs.dadi.cloud/api#authentication) section of the API documentation for full details.
 
 #### Creating the first user
 
@@ -144,11 +144,11 @@ $ npm test
 
 
 ## Links
-* [API Documentation](https://docs.dadi.tech/#api/)
+* [API Documentation](https://docs.dadi.cloud/api/)
 
 ## Contributors
 
-DADI API is based on an original idea by Joseph Denne. It is developed and maintained by the engineering team at DADI ([https://dadi.tech](https://dadi.tech))
+DADI API is based on an original idea by Joseph Denne. It is developed and maintained by the engineering team at DADI ([https://dadi.cloud](https://dadi.cloud))
 
 * Adam K Dean <[email protected]>
 * Arthur Mingard <[email protected]>
@@ -167,7 +167,7 @@ DADI API is based on an original idea by Joseph Denne. It is developed and maint
 DADI is a data centric development and delivery stack, built specifically in support of the principles of API first and COPE.
 
 Copyright notice<br />
-(C) 2018 DADI+ Limited <support@dadi.tech><br />
+(C) 2018 DADI+ Limited <support@dadi.cloud><br />
 All rights reserved
 
 This product is part of DADI.<br />

config.js

@@ -307,13 +307,13 @@ var conf = convict({
     },
     s3: {
       accessKey: {
-        doc: 'The AWS access key used to connect to S3',
+        doc: 'The access key used to connect to an S3-compatible storage provider',
         format: String,
         default: '',
         env: 'AWS_S3_ACCESS_KEY'
       },
       secretKey: {
-        doc: 'The AWS secret key used to connect to S3',
+        doc: 'The secret key used to connect to an S3-compatible storage provider',
         format: String,
         default: '',
         env: 'AWS_S3_SECRET_KEY'
@@ -325,10 +325,15 @@ var conf = convict({
         env: 'AWS_S3_BUCKET_NAME'
       },
       region: {
-        doc: 'The AWS region',
+        doc: 'The region for an S3-compatible storage provider',
         format: String,
         default: '',
         env: 'AWS_S3_REGION'
+      },
+      endpoint: {
+        doc: 'The endpoint for an S3-compatible storage provider',
+        format: String,
+        default: ''
       }
     }
   },

dadi/lib/controller/media.js

@@ -1,20 +1,18 @@
 'use strict'
 
 const Busboy = require('busboy')
+const config = require('../../../config')
+const Controller = require('./index')
+const help = require('../help')
 const imagesize = require('imagesize')
+const mediaModel = require('../model/media')
+const mime = require('mime')
 const PassThrough = require('stream').PassThrough
 const path = require('path')
-const serveStatic = require('serve-static')
 const sha1 = require('sha1')
-const url = require('url')
-
-const config = require(path.join(__dirname, '/../../../config'))
-const help = require(path.join(__dirname, '/../help'))
+const StorageFactory = require('../storage/factory')
 const streamifier = require('streamifier')
-
-const Controller = require('./index')
-const mediaModel = require(path.join(__dirname, '/../model/media'))
-const StorageFactory = require(path.join(__dirname, '/../storage/factory'))
+const url = require('url')
 
 const MediaController = function (model) {
   this.model = model
@@ -78,18 +76,12 @@ MediaController.prototype.count = function (req, res, next) {
 }
 
 /**
- * Serve a media file from its location on disk.
+ * Serve a media file from its location.
  */
 MediaController.prototype.getFile = function (req, res, next, route) {
-  // `serveStatic` will look at the entire URL to find the file it needs to
-  // serve, but we're not serving files from the root. To get around this, we
-  // pass it a modified version of the URL, where the root URL becomes just the
-  // filename parameter.
-  const modifiedReq = Object.assign({}, req, {
-    url: `${route}/${req.params.filename}`
-  })
+  let storageHandler = StorageFactory.create(req.params.filename)
 
-  return serveStatic(config.get('media.basePath'))(modifiedReq, res, next)
+  return storageHandler.get(req.params.filename, route, req, res, next)
 }
 
 /**
@@ -186,12 +178,13 @@ MediaController.prototype.post = function (req, res, next) {
         }
 
         let fields = Object.keys(this.model.schema)
+
         let obj = {
           fileName: this.fileName
         }
 
         if (fields.includes('mimetype')) {
-          obj.mimetype = this.mimetype
+          obj.mimetype = mime.getType(this.fileName)
         }
 
         // Is `imageInfo` available?
@@ -211,22 +204,26 @@ MediaController.prototype.post = function (req, res, next) {
           _createdBy: req.client && req.client.clientId
         }
 
-        const callback = (err, response) => {
-          response.results = response.results.map(document => {
-            return mediaModel.formatDocuments(document)
-          })
-
-          help.sendBackJSON(201, res, next)(err, response)
-        }
-
         return this.writeFile(req, this.fileName, this.mimetype, dataStream).then(result => {
           if (fields.includes('contentLength')) {
             obj.contentLength = result.contentLength
           }
 
           obj.path = result.path
 
-          this.model.create(obj, internals, callback, req)
+          return this.model.create({
+            documents: obj,
+            internals,
+            req
+          }).then(response => {
+            response.results = response.results.map(document => {
+              return mediaModel.formatDocuments(document)
+            })
+
+            help.sendBackJSON(201, res, next)(null, response)
+          })
+        }).catch(err => {
+          help.sendBackJSON(null, res, next)(err)
         })
       })
     })
@@ -256,6 +253,51 @@ MediaController.prototype.post = function (req, res, next) {
   }
 }
 
+MediaController.prototype.delete = function (req, res, next) {
+  let query = req.params.id ? { _id: req.params.id } : req.body.query
+
+  if (!query) return next()
+
+  this.model.get({
+    query, req
+  }).then(results => {
+    if (!results.results[0]) return next()
+
+    let file = results.results[0]
+
+    // remove physical file
+    let storageHandler = StorageFactory.create(file.fileName)
+
+    storageHandler.delete(file)
+      .then(result => {
+        this.model.delete({
+          query,
+          req
+        }).then(({deletedCount, totalCount}) => {
+          if (config.get('feedback')) {
+            // Send 200 with JSON payload.
+            return help.sendBackJSON(200, res, next)(null, {
+              status: 'success',
+              message: 'Document deleted successfully',
+              deleted: deletedCount,
+              totalCount
+            })
+          }
+
+          // Send 204 with no content.
+          res.statusCode = 204
+          res.end()
+        }).catch(error => {
+          return help.sendBackJSON(200, res, next)(error)
+        })
+      }).catch(err => {
+        return next(err)
+      })
+  }).catch(err => {
+    return next(err)
+  })
+}
+
 /**
  *
  */

dadi/lib/index.js

@@ -1088,6 +1088,7 @@ Server.prototype.addComponent = function (options) {
 
     this.components[mediaRoute] = options.component
     this.components[mediaRoute + '/:token?'] = options.component
+    this.components[mediaRoute + '/' + idParam] = options.component
     this.components[mediaRoute + '/:filename(.*png|.*jpg|.*jpeg|.*gif|.*bmp|.*tiff|.*pdf)'] = options.component
 
     if (options.component.setRoute) {
@@ -1157,7 +1158,7 @@ Server.prototype.addComponent = function (options) {
 
     // GET media
     this.app.use(mediaRoute, (req, res, next) => {
-      var method = req.method && req.method.toLowerCase()
+      let method = req.method && req.method.toLowerCase()
       if (method !== 'get') return next()
 
       if (options.component[method]) {
@@ -1171,6 +1172,16 @@ Server.prototype.addComponent = function (options) {
         return options.component.getFile(req, res, next, mediaRoute)
       }
     })
+
+    // DELETE media
+    this.app.use(`${mediaRoute}/${idParam}`, (req, res, next) => {
+      let method = req.method && req.method.toLowerCase()
+      if (method !== 'delete') return next()
+
+      if (options.component[method]) {
+        return options.component[method](req, res, next)
+      }
+    })
   }
 }
 

dadi/lib/model/media.js

@@ -68,7 +68,7 @@ MediaModel.prototype.formatDocuments = function (documents) {
 
     // Is this a relative path to a file in the disk? If so, we need to prepend
     // the API URL.
-    if (formattedDocument.path.indexOf('/') === 0) {
+    if (formattedDocument.path && formattedDocument.path.indexOf('/') === 0) {
       formattedDocument.url = this.getURLForPath(formattedDocument.path)
     }
 

dadi/lib/search/index.js

@@ -1,8 +1,7 @@
-var _ = require('underscore')
-var path = require('path')
-var url = require('url')
-var help = require(path.join(__dirname, '/../help'))
-var model = require(path.join(__dirname, '/../model'))
+const path = require('path')
+const url = require('url')
+const help = require(path.join(__dirname, '/../help'))
+const model = require(path.join(__dirname, '/../model'))
 /*
 
 Search middleware allowing cross-collection querying
@@ -17,55 +16,52 @@ http://api.example.com/1.0/search?collections=library/books,library/films&query=
 
 */
 module.exports = function (server) {
-  server.app.use('/:version/search', function (req, res, next) {
-    // sorry, we only process GET requests at this endpoint
-    var method = req.method && req.method.toLowerCase()
-    if (method !== 'get') {
+  server.app.use('/:version/search', (req, res, next) => {
+    if (req.method && req.method.toLowerCase() !== 'get') {
       return next()
     }
 
-    var path = url.parse(req.url, true)
-    var options = path.query
+    let parsedUrl = url.parse(req.url, true)
+    let options = parsedUrl.query
 
     // no collection and no query params
     if (!(options.collections && options.query)) {
       return help.sendBackJSON(400, res, next)(null, {'error': 'Bad Request'})
     }
 
     // split the collections param
-    var collections = options.collections.split(',')
+    let collections = options.collections.split(',')
 
     // extract the query from the querystring
-    var query = help.parseQuery(options.query)
+    let query = help.parseQuery(options.query)
 
     // determine API version
-    var apiVersion = path.pathname.split('/')[1]
+    let apiVersion = parsedUrl.pathname.split('/')[1]
 
     // no collections specfied
     if (collections.length === 0) {
       return help.sendBackJSON(400, res, next)(null, {'error': 'Bad Request'})
     }
 
-    var results = {}
-    var idx = 0
+    let results = {}
+    let idx = 0
 
-    _.each(collections, function (collection) {
+    collections.forEach(collection => {
       // get the database and collection name from the
       // collection parameter
-      var parts = collection.split('/')
-      var database, name, mod
+      let parts = collection.split('/')
+      let database, name, mod
 
       query._apiVersion = apiVersion
 
-      if (_.isArray(parts) && parts.length > 1) {
+      if (Array.isArray(parts) && parts.length > 1) {
         database = parts[0]
         name = parts[1]
         mod = model(name, null, null, database)
       }
 
       if (mod) {
-        // query!
-        mod.find(query, function (err, docs) {
+        mod.find(query, (err, docs) => {
           if (err) {
             return help.sendBackJSON(500, res, next)(err)
           }