Merge branch 'main' of github.com:theRealPadster/diffbot-api-node

README.md

@@ -12,13 +12,13 @@ Diffbot-API-Node is a Promise-based library to use the [Diffbot](https://www.dif
 ## Features
 
 Currently supports the following features:
-* [Analyze](#analyze-api) (with HTML POST support)
-* [Article](#article-api) (with HTML and plaintext POST support)
-* [Discussion](#discussion-api) (with HTML POST support)
-* [Event](#event-api-beta) (beta) (with HTML POST support)
-* [Image](#image-api) (with HTML POST support)
-* [Product](#product-api) (with HTML POST support)
-* [Video](#video-api) (with HTML POST support)
+* [Analyze](#analyze-api)
+* [Article](#article-api)
+* [Discussion](#discussion-api)
+* [Event](#event-api-beta) (beta)
+* [Image](#image-api)
+* [Product](#product-api)
+* [Video](#video-api)
 * [Knowledge Graph](#knowledge-graph-api)
 * [Crawl](#crawl-api)
   * New (all params supported except `customHeaders`)

package.json

@@ -6,7 +6,7 @@
     "api",
     "node"
   ],
-  "version": "0.4.2",
+  "version": "0.9.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/therealpadster/diffbot-api-node.git"

src/lib/request.d.ts

@@ -1,6 +1,6 @@
-export function generate(url: string, method?: string, body?: string, customHeaders?: any): Request;
+export function generate(url: string, method?: string, body?: string, customHeaders?: object): Request;
 export function exec(request: Request): any;
 /**
  * Generate a simple request object
  */
-export type Request = any;
+export type Request = object;

src/lib/request.js

@@ -3,28 +3,33 @@ const axios = require('axios');
 // TODO: figure out how to get JSDocs to work properly
 /**
  * Generate a simple request object
- * @typedef {Object} Request
+ * @typedef {object} Request
  * @param {string} url The URL
  * @param {string} method The HTTP method to use (defaults to GET)
  * @param {string} [body] Optional HTML markup or plaintext to pass as POST body
- * @param {Object} headers The request headers
+ * @param {object} headers The request headers
  */
 
 /**
  * Generate a simple request object
  * @param {string} url The URL
  * @param {string} [method] The HTTP method to use (defaults to GET)
  * @param {string} [body] Optional HTML markup or plaintext to pass as POST body
- * @param {Object} [customHeaders] Optional additional request headers object
+ * @param {object} [customHeaders] Optional additional request headers object
  * @returns {Request} The request object
  */
 exports.generate = function(url, method = 'GET', body, customHeaders) {
-  let headers = { ...customHeaders };
+  // Add 'X-Forward-' to all the custom headers
+  const headers = Object.entries({...customHeaders}).reduce((acc, [key, value]) => {
+    acc['X-Forward-' + key] = value;
+    return acc;
+  }, {});
+
   if (body) {
     if (body.startsWith('<'))
-      headers = { 'Content-Type': 'text/html' };
+      headers['Content-Type'] = 'text/html';
     else
-      headers = { 'Content-Type': 'text/plain' };
+      headers['Content-Type'] = 'text/plain';
   }
 
   return {

test/analyze.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customHeaders, customJS } = require('./global');
 
 describe('Analyze Tests', function() {
 
@@ -24,16 +24,17 @@ describe('Analyze Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the analyze GET request with custom JS', async () => {
+  it('should generate the analyze GET request with custom headers', async () => {
     const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
 
-    let request = await diffbot.analyze({ url, customJS });
+    let request = await diffbot.analyze({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/analyze?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/article.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Article Tests', function() {
 
@@ -25,16 +25,17 @@ describe('Article Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the article GET request with custom JS', async () => {
+  it('should generate the article GET request with custom headers', async () => {
     const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
 
-    let request = await diffbot.article({ url, customJS });
+    let request = await diffbot.article({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/article?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/discussion.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Discussion Tests', function() {
 
@@ -21,16 +21,17 @@ describe('Discussion Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the discussion GET request with custom JS', async () => {
+  it('should generate the discussion GET request with custom headers', async () => {
     const url = 'https://www.theverge.com/2020/8/25/21400240/epic-apple-ruling-unreal-engine-fortnite-temporary-restraining-order';
 
-    let request = await diffbot.discussion({ url, customJS });
+    let request = await diffbot.discussion({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/discussion?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/event.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Event Tests', function() {
 
@@ -21,14 +21,15 @@ describe('Event Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the event GET request with custom JS', async () => {
-    let request = await diffbot.event({ url, customJS });
+  it('should generate the event GET request with custom headers', async () => {
+    let request = await diffbot.event({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/event?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/global.js

@@ -3,8 +3,8 @@ const chai = require('chai');
 const chaiAsPromised = require('chai-as-promised');
 chai.use(chaiAsPromised);
 
-function start(){};
-function end(){};
+function start(){}
+function end(){}
 const customJS = function() {
   start();
   setTimeout(function() {
@@ -18,9 +18,10 @@ const customJS = function() {
       end();
     }
   }, 500);
-}.toString();
+};
 
 const FAKE_TOKEN = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx';
 exports.diffbot = new Diffbot(FAKE_TOKEN, true);
 exports.expect = chai.expect;
-exports.customJS = customJS;
+exports.customJS = customJS.toString();
+exports.customHeaders = { 'User-Agent': 'Diffbot' };

test/image.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Image Tests', function() {
 
@@ -20,16 +20,17 @@ describe('Image Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the image GET request with custom JS', async () => {
+  it('should generate the image GET request with custom headers', async () => {
     const url = 'https://www.deviantart.com/up-tchi/art/Coral-village-852927725';
 
-    let request = await diffbot.image({ url, customJS });
+    let request = await diffbot.image({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/image?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/product.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Product Tests', function() {
 
@@ -35,14 +35,15 @@ describe('Product Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the product GET request with custom JS', async () => {
-    let request = await diffbot.product({ url, customJS });
+  it('should generate the product GET request with custom headers', async () => {
+    let request = await diffbot.product({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/product?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });

test/video.test.js

@@ -1,4 +1,4 @@
-const { diffbot, expect, customJS } = require('./global');
+const { diffbot, expect, customJS, customHeaders } = require('./global');
 
 describe('Video Tests', function() {
 
@@ -20,16 +20,17 @@ describe('Video Tests', function() {
     return Promise.resolve(true);
   });
 
-  it('should generate the video GET request with custom JS', async () => {
+  it('should generate the video GET request with custom headers', async () => {
     const url = 'https://www.youtube.com/watch?v=HeiPdaTQTfo';
 
-    let request = await diffbot.video({ url, customJS });
+    let request = await diffbot.video({ url, customJS, customHeaders });
 
     expect(request.url).to.equal(`https://api.diffbot.com/v3/video?token=${diffbot.token}&url=${encodeURIComponent(url)}`);
     expect(request.method).to.equal('GET');
     expect(request.body).to.be.undefined;
     expect(request.headers).to.be.an('object');
     expect(request.headers['X-Forward-X-Evaluate']).to.equal(customJS.replace(/(\r?\n|\r)\s+/g, ''));
+    expect(request.headers['X-Forward-User-Agent']).to.equal(customHeaders['User-Agent']);
 
     return Promise.resolve(true);
   });