From 38b2317fdf8c285e4740bd52a615edfdd7c071bc Mon Sep 17 00:00:00 2001
From: Remy <muryoh@gmail.com>
Date: Mon, 6 Nov 2023 11:33:44 +0100
Subject: [PATCH] pnpm lock + cdn includes + data-abovethefold

---
 pnpm-lock.yaml        |  7 +++++++
 src/config-default.ts |  2 ++
 src/config-types.ts   |  2 ++
 src/optimize.ts       | 48 +++++++++++++++++++++++++++++++++++--------
 4 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 784e8de..7b797b0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -68,6 +68,9 @@ dependencies:
   undici:
     specifier: ^5.24.0
     version: 5.24.0
+  unpic:
+    specifier: ^3.11.0
+    version: 3.12.0
 
 devDependencies:
   '@types/html-minifier-terser':
@@ -1956,6 +1959,10 @@ packages:
       busboy: 1.6.0
     dev: false
 
+  /unpic@3.12.0:
+    resolution: {integrity: sha512-dmp0XRBeE6i+wWlWAkF7ordJYi5y7neEGSNvlM+92HUp1VnLUb37NVUZI6AZgb2Rf/vZynSadZkdGbPGXc/ZIA==}
+    dev: false
+
   /uri-js@4.4.1:
     resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
     dependencies:
diff --git a/src/config-default.ts b/src/config-default.ts
index f2dca46..dd5b36d 100644
--- a/src/config-default.ts
+++ b/src/config-default.ts
@@ -22,6 +22,8 @@ const default_options: Options = {
     },
     cdn: {
       process: 'off',
+      src_include: /^.*$/,
+      src_exclude: null,
     },
     compress: true,
     jpeg: {
diff --git a/src/config-types.ts b/src/config-types.ts
index bcfee46..894b0b8 100644
--- a/src/config-types.ts
+++ b/src/config-types.ts
@@ -36,6 +36,8 @@ export type Options = {
       process:
         | 'off' //default
         | 'optimize';
+      src_include: RegExp;
+      src_exclude: RegExp | null;
     };
     compress: boolean;
     jpeg: {
diff --git a/src/optimize.ts b/src/optimize.ts
index b8f25a4..7903c3c 100755
--- a/src/optimize.ts
+++ b/src/optimize.ts
@@ -21,6 +21,7 @@ import { inlineCriticalCss } from './optimizers/inline-critical-css.js';
 import { prefetch_links_in_viewport } from './optimizers/prefetch-links.js';
 
 const UNPIC_DEFAULT_HOST_REGEX = /^https:\/\/n\//g;
+const ABOVE_FOLD_DATA_ATTR = 'data-abovethefold';
 
 function getIntAttr(
   img: cheerio.Cheerio<cheerio.Element>,
@@ -57,7 +58,8 @@ async function analyse(file: string): Promise<void> {
     );
 
     const img = $(imgElement);
-    const isAboveTheFold = imgElement.startIndex! < theFold;
+    const isAboveTheFold = isImgAboveTheFold(img, imgElement, theFold);
+    img.removeAttr(ABOVE_FOLD_DATA_ATTR);
 
     try {
       await processImage(file, img, isAboveTheFold);
@@ -185,6 +187,20 @@ async function analyse(file: string): Promise<void> {
   }
 }
 
+function isImgAboveTheFold(
+  img: cheerio.Cheerio<cheerio.Element>,
+  imgElement: cheerio.Element,
+  theFold: number
+) {
+  const aboveTheFoldAttr: string | number | undefined =
+    img.attr(ABOVE_FOLD_DATA_ATTR);
+  if (aboveTheFoldAttr) {
+    const parsed = parseInt(aboveTheFoldAttr);
+    if (!Number.isNaN(parsed)) return !!parsed;
+  }
+  return imgElement.startIndex! < theFold;
+}
+
 function getTheFold($: cheerio.CheerioAPI): number {
   const theFolds = $('the-fold');
 
@@ -292,13 +308,7 @@ async function processImage(
   /*
    * Check for external images
    */
-  if (
-    !isLocal(attrib_src) &&
-    !!config.image.external.src_include &&
-    attrib_src.match(config.image.external.src_include) &&
-    (!config.image.external.src_exclude ||
-      !attrib_src.match(config.image.external.src_exclude))
-  ) {
+  if (!isLocal(attrib_src)) {
     switch (config.image.cdn.process) {
       case 'off':
         break;
@@ -307,6 +317,14 @@ async function processImage(
         if (!canonical) break;
         const cdnTransformer = getTransformer(canonical.cdn);
         if (!cdnTransformer) break;
+        if (
+          !isIncluded(
+            attrib_src,
+            config.image.cdn.src_include,
+            config.image.cdn.src_exclude
+          )
+        )
+          break;
         let attrib_width = getIntAttr(img, 'width');
         if (!attrib_width) {
           $state.reportIssue(htmlfile, {
@@ -344,6 +362,14 @@ async function processImage(
       case 'off': // Don't process external images
         return;
       case 'download': // Download external image for local processing
+        if (
+          !isIncluded(
+            attrib_src,
+            config.image.external.src_include,
+            config.image.external.src_exclude
+          )
+        )
+          break;
         try {
           attrib_src = await downloadExternalImage(htmlfile, attrib_src);
           img.attr('src', attrib_src);
@@ -656,6 +682,12 @@ async function processImage(
   }
 }
 
+const isIncluded = (
+  src: string,
+  includeConf: RegExp,
+  excludeConf: RegExp | null
+) => !!src.match(includeConf) && (!excludeConf || !src.match(excludeConf));
+
 async function _generateSrcSet(
   startSrc: string | undefined,
   imageWidth: number | undefined,