Resource Hints 2021 query amends

sql/2021/resource-hints/consoleLog_incorrect_crossorigin.sql

@@ -0,0 +1,48 @@
+#standardSQL
+# returns the number of pages using preload tags without the required crossorigin attribute
+
+CREATE TEMPORARY FUNCTION getResourceHints(payload STRING)
+RETURNS STRUCT<preload BOOLEAN, prefetch BOOLEAN, preconnect BOOLEAN, prerender BOOLEAN, `dns-prefetch` BOOLEAN, `modulepreload` BOOLEAN>
+LANGUAGE js AS '''
+var hints = ['preload', 'prefetch', 'preconnect', 'prerender', 'dns-prefetch', 'modulepreload'];
+try {
+  var $ = JSON.parse(payload);
+  var almanac = JSON.parse($._almanac);
+  return hints.reduce((results, hint) => {
+    results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint);
+    return results;
+  }, {});
+} catch (e) {
+  return hints.reduce((results, hint) => {
+    results[hint] = false;
+    return results;
+  }, {});
+}
+''' ;
+
+SELECT
+  client,
+  ARRAY_LENGTH(value) AS num_incorrect_crossorigin,
+  COUNT(0) AS freq,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
+FROM (
+  SELECT
+    client,
+    REGEXP_EXTRACT_ALL(consoleLog, r'A preload for (.+?) is found, but is not used because the request credentials mode does not match') AS value
+  FROM (
+    SELECT
+      _TABLE_SUFFIX AS client,
+      JSON_EXTRACT(payload, "$._consoleLog") AS consoleLog,
+      getResourceHints(payload) AS hints
+    FROM
+      `httparchive.pages.2021_07_01_*`
+  )
+  WHERE hints.preload
+)
+GROUP BY
+  client,
+  num_incorrect_crossorigin
+ORDER BY
+  client,
+  freq DESC

sql/2021/resource-hints/consoleLog_incorrect_type.sql

@@ -1,9 +1,28 @@
 #standardSQL
 # returns the number of pages which preload a resource of the incorrect script type
 
+CREATE TEMPORARY FUNCTION getResourceHints(payload STRING)
+RETURNS STRUCT<preload BOOLEAN, prefetch BOOLEAN, preconnect BOOLEAN, prerender BOOLEAN, `dns-prefetch` BOOLEAN, `modulepreload` BOOLEAN>
+LANGUAGE js AS '''
+var hints = ['preload', 'prefetch', 'preconnect', 'prerender', 'dns-prefetch', 'modulepreload'];
+try {
+  var $ = JSON.parse(payload);
+  var almanac = JSON.parse($._almanac);
+  return hints.reduce((results, hint) => {
+    results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint);
+    return results;
+  }, {});
+} catch (e) {
+  return hints.reduce((results, hint) => {
+    results[hint] = false;
+    return results;
+  }, {});
+}
+''' ;
+
 SELECT
   client,
-  ARRAY_LENGTH(value) AS numOfIncorrectType,
+  ARRAY_LENGTH(value) AS num_incorrect_type,
   COUNT(0) AS freq,
   SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
   COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
@@ -14,14 +33,16 @@ FROM (
   FROM (
     SELECT
       _TABLE_SUFFIX AS client,
-      JSON_EXTRACT(payload, "$._consoleLog") AS consoleLog
+      JSON_EXTRACT(payload, "$._consoleLog") AS consoleLog,
+      getResourceHints(payload) AS hints
     FROM
       `httparchive.pages.2021_07_01_*`
   )
+  WHERE hints.preload
 )
 GROUP BY
   client,
-  numOfIncorrectType
+  num_incorrect_type
 ORDER BY
   client,
-  numOfIncorrectType
+  num_incorrect_type

sql/2021/resource-hints/consoleLog_unused_preload.sql

@@ -1,9 +1,28 @@
 #standardSQL
 # returns the number of unused preloaded resources
 
+CREATE TEMPORARY FUNCTION getResourceHints(payload STRING)
+RETURNS STRUCT<preload BOOLEAN, prefetch BOOLEAN, preconnect BOOLEAN, prerender BOOLEAN, `dns-prefetch` BOOLEAN, `modulepreload` BOOLEAN>
+LANGUAGE js AS '''
+var hints = ['preload', 'prefetch', 'preconnect', 'prerender', 'dns-prefetch', 'modulepreload'];
+try {
+  var $ = JSON.parse(payload);
+  var almanac = JSON.parse($._almanac);
+  return hints.reduce((results, hint) => {
+    results[hint] = !!almanac['link-nodes'].nodes.find(link => link.rel.toLowerCase() == hint);
+    return results;
+  }, {});
+} catch (e) {
+  return hints.reduce((results, hint) => {
+    results[hint] = false;
+    return results;
+  }, {});
+}
+''' ;
+
 SELECT
   client,
-  ARRAY_LENGTH(value) AS numOfUnusedPreloads,
+  ARRAY_LENGTH(value) AS num_unused_preload,
   COUNT(0) AS freq,
   SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
   COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
@@ -14,14 +33,16 @@ FROM (
   FROM (
     SELECT
       _TABLE_SUFFIX AS client,
-      JSON_EXTRACT(payload, "$._consoleLog") AS consoleLog
+      JSON_EXTRACT(payload, "$._consoleLog") AS consoleLog,
+      getResourceHints(payload) AS hints
     FROM
       `httparchive.pages.2021_07_01_*`
   )
+  WHERE hints.preload
 )
 GROUP BY
   client,
-  numOfUnusedPreloads
+  num_unused_preload
 ORDER BY
   client,
-  numOfUnusedPreloads
+  num_unused_preload

sql/2021/resource-hints/preload_imagesizes_imagesrcset_adoption.sql

@@ -0,0 +1,51 @@
+#standardSQL
+# Attribute popularity for imagesrcset and imagesizes on rel="preload"
+
+CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING)
+RETURNS ARRAY<STRUCT<name STRING, attribute STRING, value STRING>>
+LANGUAGE js AS '''
+var hints = new Set(['preload']);
+var attributes = ['imagesrcset', 'imagesizes'];
+try {
+  var $ = JSON.parse(payload);
+  var almanac = JSON.parse($._almanac);
+  return almanac['link-nodes'].nodes.reduce((results, link) => {
+    var hint = link.rel.toLowerCase();
+    if (!hints.has(hint)) {
+      return results;
+    }
+    attributes.forEach(attribute => {
+      var value = link[attribute];
+      results.push({
+        name: hint,
+        attribute: attribute,
+        // Support empty strings.
+        value: typeof value == 'string' ? value : null
+      });
+    });
+    return results;
+  }, []);
+} catch (e) {
+  return [];
+}
+''' ;
+
+SELECT
+  _TABLE_SUFFIX AS client,
+  hint.name AS name,
+  hint.attribute AS attribute,
+  COUNTIF(hint.value IS NOT NULL) AS freq,
+  SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX, hint.name) AS total,
+  COUNTIF(hint.value IS NOT NULL) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX, hint.name) AS pct
+FROM
+  `httparchive.pages.2021_07_01_*`,
+  UNNEST(getResourceHintAttrs(payload)) AS hint
+GROUP BY
+  client,
+  name,
+  attribute
+ORDER BY
+  client,
+  name,
+  attribute,
+  pct DESC