From 6882337d1954a00da72af2a89ea1b7bbf23d16ed Mon Sep 17 00:00:00 2001 From: Axel Vermeil Date: Tue, 7 Jul 2020 14:25:14 +0000 Subject: [PATCH] hack: print less of the page logs to avoid filling server logs --- src/core/render-core.js | 213 ++++++++++++++++++++-------------------- 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/src/core/render-core.js b/src/core/render-core.js index 597d7548..3224a77a 100644 --- a/src/core/render-core.js +++ b/src/core/render-core.js @@ -1,7 +1,7 @@ -const puppeteer = require('puppeteer') -const _ = require('lodash') -const config = require('../config') -const logger = require('../util/logger')(__filename) +const puppeteer = require("puppeteer"); +const _ = require("lodash"); +const config = require("../config"); +const logger = require("../util/logger")(__filename); async function render(_opts = {}) { const opts = _.merge( @@ -13,213 +13,212 @@ async function render(_opts = {}) { html: null, viewport: { width: 1600, - height: 1200 + height: 1200, }, goto: { - waitUntil: 'networkidle2' + waitUntil: "networkidle2", }, - output: 'pdf', + output: "pdf", pdf: { - format: 'A4', - printBackground: true + format: "A4", + printBackground: true, }, screenshot: { - type: 'png', - fullPage: true + type: "png", + fullPage: true, }, - failEarly: false + failEarly: false, }, _opts - ) + ); - if (_.get(_opts, 'pdf.width') && _.get(_opts, 'pdf.height')) { + if (_.get(_opts, "pdf.width") && _.get(_opts, "pdf.height")) { // pdf.format always overrides width and height, so we must delete it // when user explicitly wants to set width and height - opts.pdf.format = undefined + opts.pdf.format = undefined; } - logOpts(opts) + logOpts(opts); const browser = await puppeteer.launch({ headless: !config.DEBUG_MODE, ignoreHTTPSErrors: opts.ignoreHttpsErrors, - args: ['--disable-gpu', '--no-sandbox', '--disable-setuid-sandbox', '--font-render-hinting=none'], - sloMo: config.DEBUG_MODE ? 250 : undefined - }) - const page = await browser.newPage() - - page.on('console', (...args) => logger.info('PAGE LOG:', ...args)) - - page.on('error', (err) => { - logger.error(`Error event emitted: ${err}`) - logger.error(err.stack) - browser.close() - }) - - this.failedResponses = [] - page.on('requestfailed', (request) => { - this.failedResponses.push(request) + args: [ + "--disable-gpu", + "--no-sandbox", + "--disable-setuid-sandbox", + "--font-render-hinting=none", + ], + sloMo: config.DEBUG_MODE ? 250 : undefined, + }); + const page = await browser.newPage(); + + page.on("console", (...args) => logger.info("PAGE LOG:", args[0]._text)); + + page.on("error", (err) => { + logger.error(`Error event emitted: ${err}`); + logger.error(err.stack); + browser.close(); + }); + + this.failedResponses = []; + page.on("requestfailed", (request) => { + this.failedResponses.push(request); if (request.url === opts.url) { - this.mainUrlResponse = request + this.mainUrlResponse = request; } - }) + }); - page.on('response', (response) => { + page.on("response", (response) => { if (response && response.status >= 400) { - this.failedResponses.push(response) + this.failedResponses.push(response); } if (response.url === opts.url) { - this.mainUrlResponse = response + this.mainUrlResponse = response; } - }) + }); - let data + let data; try { - logger.info('Set browser viewport..') - await page.setViewport(opts.viewport) + logger.info("Set browser viewport.."); + await page.setViewport(opts.viewport); if (opts.emulateScreenMedia) { - logger.info('Emulate @media screen..') - await page.emulateMedia('screen') + logger.info("Emulate @media screen.."); + await page.emulateMedia("screen"); } if (opts.cookies && opts.cookies.length > 0) { - logger.info('Setting cookies..') + logger.info("Setting cookies.."); - const client = await page.target().createCDPSession() + const client = await page.target().createCDPSession(); - await client.send('Network.enable') - await client.send('Network.setCookies', { cookies: opts.cookies }) + await client.send("Network.enable"); + await client.send("Network.setCookies", { cookies: opts.cookies }); } if (opts.html) { - logger.info('Set HTML ..') - await page.setContent(opts.html, opts.goto) + logger.info("Set HTML .."); + await page.setContent(opts.html, opts.goto); } else { - logger.info(`Goto url ${opts.url} ..`) - await page.goto(opts.url, opts.goto) + logger.info(`Goto url ${opts.url} ..`); + await page.goto(opts.url, opts.goto); } if (_.isNumber(opts.waitFor) || _.isString(opts.waitFor)) { - logger.info(`Wait for ${opts.waitFor} ..`) - await page.waitFor(opts.waitFor) + logger.info(`Wait for ${opts.waitFor} ..`); + await page.waitFor(opts.waitFor); } if (opts.scrollPage) { - logger.info('Scroll page ..') - await scrollPage(page) + logger.info("Scroll page .."); + await scrollPage(page); } if (this.failedResponses.length) { - logger.warn(`Number of failed requests: ${this.failedResponses.length}`) + logger.warn(`Number of failed requests: ${this.failedResponses.length}`); this.failedResponses.forEach((response) => { - logger.warn(`${response.status} ${response.url}`) - }) + logger.warn(`${response.status} ${response.url}`); + }); - if (opts.failEarly === 'all') { + if (opts.failEarly === "all") { const err = new Error( - `${ - this.failedResponses.length - } requests have failed. See server log for more details.` - ) - err.status = 412 - throw err + `${this.failedResponses.length} requests have failed. See server log for more details.` + ); + err.status = 412; + throw err; } } if ( - opts.failEarly === 'page' && + opts.failEarly === "page" && this.mainUrlResponse && this.mainUrlResponse.status !== 200 ) { - const msg = `Request for ${ - opts.url - } did not directly succeed and returned status ${ - this.mainUrlResponse.status - }` - const err = new Error(msg) - err.status = 412 - throw err + const msg = `Request for ${opts.url} did not directly succeed and returned status ${this.mainUrlResponse.status}`; + const err = new Error(msg); + err.status = 412; + throw err; } - logger.info('Rendering ..') + logger.info("Rendering .."); if (config.DEBUG_MODE) { const msg = `\n\n---------------------------------\n Chrome does not support rendering in "headed" mode. See this issue: https://github.com/GoogleChrome/puppeteer/issues/576 \n---------------------------------\n\n - ` - throw new Error(msg) + `; + throw new Error(msg); } - if (opts.output === 'pdf') { - data = await page.pdf(opts.pdf) + if (opts.output === "pdf") { + data = await page.pdf(opts.pdf); } else { // This is done because puppeteer throws an error if fullPage and clip is used at the same // time even though clip is just empty object {} - const screenshotOpts = _.cloneDeep(_.omit(opts.screenshot, ['clip'])) + const screenshotOpts = _.cloneDeep(_.omit(opts.screenshot, ["clip"])); const clipContainsSomething = _.some( opts.screenshot.clip, (val) => !_.isUndefined(val) - ) + ); if (clipContainsSomething) { - screenshotOpts.clip = opts.screenshot.clip + screenshotOpts.clip = opts.screenshot.clip; } - data = await page.screenshot(screenshotOpts) + data = await page.screenshot(screenshotOpts); } } catch (err) { - logger.error(`Error when rendering page: ${err}`) - logger.error(err.stack) - throw err + logger.error(`Error when rendering page: ${err}`); + logger.error(err.stack); + throw err; } finally { - logger.info('Closing browser..') + logger.info("Closing browser.."); if (!config.DEBUG_MODE) { - await browser.close() + await browser.close(); } } - return data + return data; } async function scrollPage(page) { // Scroll to page end to trigger lazy loading elements await page.evaluate(() => { - const scrollInterval = 100 - const scrollStep = Math.floor(window.innerHeight / 2) - const bottomThreshold = 400 + const scrollInterval = 100; + const scrollStep = Math.floor(window.innerHeight / 2); + const bottomThreshold = 400; function bottomPos() { - return window.pageYOffset + window.innerHeight + return window.pageYOffset + window.innerHeight; } return new Promise((resolve, reject) => { function scrollDown() { - window.scrollBy(0, scrollStep) + window.scrollBy(0, scrollStep); if (document.body.scrollHeight - bottomPos() < bottomThreshold) { - window.scrollTo(0, 0) - setTimeout(resolve, 500) - return + window.scrollTo(0, 0); + setTimeout(resolve, 500); + return; } - setTimeout(scrollDown, scrollInterval) + setTimeout(scrollDown, scrollInterval); } - setTimeout(reject, 30000) - scrollDown() - }) - }) + setTimeout(reject, 30000); + scrollDown(); + }); + }); } function logOpts(opts) { - const supressedOpts = _.cloneDeep(opts) + const supressedOpts = _.cloneDeep(opts); if (opts.html) { - supressedOpts.html = '...' + supressedOpts.html = "..."; } - logger.info(`Rendering with opts: ${JSON.stringify(supressedOpts, null, 2)}`) + logger.info(`Rendering with opts: ${JSON.stringify(supressedOpts, null, 2)}`); } module.exports = { - render -} + render, +};