diff --git a/sql/2021/javascript/ajax_request_per_page.sql b/sql/2021/javascript/ajax_request_per_page.sql new file mode 100644 index 00000000000..46fae7d771f --- /dev/null +++ b/sql/2021/javascript/ajax_request_per_page.sql @@ -0,0 +1,22 @@ +SELECT + client, + percentile, + APPROX_QUANTILES(ajax_requests_total, 1000)[ + OFFSET(percentile * 10)] AS ajax_requests_total +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.ajax_requests.total') AS INT64) AS ajax_requests_total + FROM + `httparchive.pages.2021_07_01_*` ), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +WHERE + ajax_requests_total > 0 +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/bytes_2019.sql b/sql/2021/javascript/bytes_2019.sql new file mode 100644 index 00000000000..3348585ad43 --- /dev/null +++ b/sql/2021/javascript/bytes_2019.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2019) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2019_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/bytes_2020.sql b/sql/2021/javascript/bytes_2020.sql new file mode 100644 index 00000000000..839c6139ca8 --- /dev/null +++ b/sql/2021/javascript/bytes_2020.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2020) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2020_08_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/bytes_2021.sql b/sql/2021/javascript/bytes_2021.sql new file mode 100644 index 00000000000..25f82a8ca4c --- /dev/null +++ b/sql/2021/javascript/bytes_2021.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2020) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2021_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/bytes_by_3p.sql b/sql/2021/javascript/bytes_by_3p.sql new file mode 100644 index 00000000000..f5bde4b9d9f --- /dev/null +++ b/sql/2021/javascript/bytes_by_3p.sql @@ -0,0 +1,33 @@ +#standardSQL +# Distribution of 1P/3P JS bytes +SELECT + percentile, + client, + host, + APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + SUM(respSize) / 1024 AS kbytes + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script' + GROUP BY + client, + page, + host), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client, + host +ORDER BY + percentile, + client, + host diff --git a/sql/2021/javascript/compression_method.sql b/sql/2021/javascript/compression_method.sql new file mode 100644 index 00000000000..273e0e49551 --- /dev/null +++ b/sql/2021/javascript/compression_method.sql @@ -0,0 +1,47 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + compression, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + compression +ORDER BY + pct_js_requests DESC diff --git a/sql/2021/javascript/compression_method_by_3p.sql b/sql/2021/javascript/compression_method_by_3p.sql new file mode 100644 index 00000000000..e9d00d1d453 --- /dev/null +++ b/sql/2021/javascript/compression_method_by_3p.sql @@ -0,0 +1,52 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + compression, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + compression +ORDER BY + pct DESC diff --git a/sql/2021/javascript/compression_none_by_bytes.sql b/sql/2021/javascript/compression_none_by_bytes.sql new file mode 100644 index 00000000000..592248c542f --- /dev/null +++ b/sql/2021/javascript/compression_none_by_bytes.sql @@ -0,0 +1,55 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + IF(kbytes < 100, FLOOR(kbytes / 5) * 5, 100) AS kbytes, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + respSize / 1024 AS kbytes + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script' AND + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') IS NULL) +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + kbytes +ORDER BY + client, + host, + kbytes diff --git a/sql/2021/javascript/frameworks-bytes-by-framework.sql b/sql/2021/javascript/frameworks-bytes-by-framework.sql new file mode 100644 index 00000000000..17d3c9a8535 --- /dev/null +++ b/sql/2021/javascript/frameworks-bytes-by-framework.sql @@ -0,0 +1,35 @@ +#standardSQL +# Sum of JS request bytes per page by framework (2020) +SELECT + percentile, + client, + app AS js_framework, + COUNT(DISTINCT page) AS pages, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + bytesJs + FROM + `httparchive.summary_pages.2021_07_01_*`) +JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url AS page, + app + FROM + `httparchive.technologies.2021_07_01_*` + WHERE + category = 'JavaScript frameworks') +USING + (client, page), + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + percentile, + client, + js_framework +ORDER BY + percentile, + client, + pages DESC diff --git a/sql/2021/javascript/frameworks_libraries.sql b/sql/2021/javascript/frameworks_libraries.sql new file mode 100644 index 00000000000..27b0627455f --- /dev/null +++ b/sql/2021/javascript/frameworks_libraries.sql @@ -0,0 +1,30 @@ +#standardSQL +# Top JS frameworks and libraries +SELECT + _TABLE_SUFFIX AS client, + category, + app, + COUNT(DISTINCT url) AS pages, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2021_07_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category IN ('JavaScript frameworks', 'JavaScript libraries') +GROUP BY + client, + category, + app, + total +ORDER BY + pct DESC diff --git a/sql/2021/javascript/frameworks_libraries_by_version.sql b/sql/2021/javascript/frameworks_libraries_by_version.sql new file mode 100644 index 00000000000..9d56b59fe2e --- /dev/null +++ b/sql/2021/javascript/frameworks_libraries_by_version.sql @@ -0,0 +1,32 @@ +#standardSQL +# Top JS frameworks and libraries by version +SELECT + _TABLE_SUFFIX AS client, + category, + app, + info AS version, + COUNT(DISTINCT url) AS pages, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2021_07_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React') +GROUP BY + client, + category, + app, + info, + total +ORDER BY + pct DESC diff --git a/sql/2021/javascript/frameworks_libraries_combos.sql b/sql/2021/javascript/frameworks_libraries_combos.sql new file mode 100644 index 00000000000..24bea43f43a --- /dev/null +++ b/sql/2021/javascript/frameworks_libraries_combos.sql @@ -0,0 +1,43 @@ +#standardSQL +# Top JS frameworks and libraries combinations +SELECT + * +FROM ( + SELECT + client, + apps, + COUNT(DISTINCT page) AS pages, + total, + COUNT(DISTINCT page) / total AS pct + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + total, + ARRAY_TO_STRING(ARRAY_AGG(app ORDER BY app), ', ') AS apps + FROM + `httparchive.technologies.2021_07_01_*` + JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + _TABLE_SUFFIX) + USING + (_TABLE_SUFFIX) + WHERE + category IN ('JavaScript frameworks', 'JavaScript libraries') + GROUP BY + client, + url, + total) + GROUP BY + client, + apps, + total) +WHERE + pages >= 10000 +ORDER BY + pct DESC diff --git a/sql/2021/javascript/h1_vs_h2_vs_h3.sql b/sql/2021/javascript/h1_vs_h2_vs_h3.sql new file mode 100644 index 00000000000..59645440827 --- /dev/null +++ b/sql/2021/javascript/h1_vs_h2_vs_h3.sql @@ -0,0 +1,39 @@ +SELECT + client, + percentile, + APPROX_QUANTILES((ajax_h1 + resources_h1), 1000)[ + OFFSET(percentile * 10)] AS h1_request, + APPROX_QUANTILES((ajax_h2 + resources_h2), 1000)[ + OFFSET(percentile * 10)] AS h2_request, + APPROX_QUANTILES((ajax_h3 + resources_h3), 1000)[ + OFFSET(percentile * 10)] AS h3_request +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.ajax_h1') AS INT64) AS ajax_h1, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.resources_h1') AS INT64) AS resources_h1, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.ajax_h2') AS INT64) AS ajax_h2, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.resources_h2') AS INT64) AS resources_h2, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.ajax_h3') AS INT64) AS ajax_h3, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.requests_protocol.resources_h3') AS INT64) AS resources_h3 + FROM + `httparchive.pages.2021_07_01_*`), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/iframe_per_page.sql b/sql/2021/javascript/iframe_per_page.sql new file mode 100644 index 00000000000..684630f0053 --- /dev/null +++ b/sql/2021/javascript/iframe_per_page.sql @@ -0,0 +1,22 @@ +SELECT + client, + percentile, + APPROX_QUANTILES(iframe_total, 1000)[ + OFFSET(percentile * 10)] AS iframe_total +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.iframe') AS INT64) AS iframe_total + FROM + `httparchive.pages.2021_07_01_*` ), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +WHERE + iframe_total > 0 +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/lighthouse_unminified_js.sql b/sql/2021/javascript/lighthouse_unminified_js.sql new file mode 100644 index 00000000000..a5e48687936 --- /dev/null +++ b/sql/2021/javascript/lighthouse_unminified_js.sql @@ -0,0 +1,18 @@ +#standardSQL +# Pages with unminified JS +SELECT + score, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER () AS total, + COUNT(0) / SUM(COUNT(0)) OVER () AS pct +FROM ( + SELECT + JSON_EXTRACT_SCALAR(report, "$.audits['unminified-javascript'].score") AS score + FROM + `httparchive.lighthouse.2021_07_01_mobile`) +WHERE + score IS NOT NULL +GROUP BY + score +ORDER BY + score diff --git a/sql/2021/javascript/lighthouse_unminified_js_by_3p.sql b/sql/2021/javascript/lighthouse_unminified_js_by_3p.sql new file mode 100644 index 00000000000..e93e86bd657 --- /dev/null +++ b/sql/2021/javascript/lighthouse_unminified_js_by_3p.sql @@ -0,0 +1,34 @@ +#standardSQL +# Pages with unminified JS by 1P/3P +CREATE TEMPORARY FUNCTION getUnminifiedJsUrls(audit STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(({url, wastedBytes}) => { + return {url, wastedBytes}; + }); +} catch (e) { + return []; +} +'''; + +SELECT + AVG(pct_1p_wasted_bytes) AS avg_pct_1p_wasted_bytes, + AVG(pct_3p_wasted_bytes) AS avg_pct_3p_wasted_bytes +FROM ( + SELECT + page, + SUM(IF(is_3p, 0, wasted_bytes)) / SUM(wasted_bytes) AS pct_1p_wasted_bytes, + SUM(IF(is_3p, wasted_bytes, 0)) / SUM(wasted_bytes) AS pct_3p_wasted_bytes + FROM ( + SELECT + test.url AS page, + NET.HOST(unminified.url) IS NOT NULL AND NET.HOST(unminified.url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting' + ) AS is_3p, + unminified.wastedBytes AS wasted_bytes + FROM + `httparchive.lighthouse.2021_07_01_mobile` AS test, + UNNEST(getUnminifiedJsUrls(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified) + GROUP BY + page) diff --git a/sql/2021/javascript/lighthouse_unminified_js_bytes.sql b/sql/2021/javascript/lighthouse_unminified_js_bytes.sql new file mode 100644 index 00000000000..923cd16e87c --- /dev/null +++ b/sql/2021/javascript/lighthouse_unminified_js_bytes.sql @@ -0,0 +1,31 @@ +#standardSQL +# Histogram of wasted bytes per page +CREATE TEMPORARY FUNCTION getUnminifiedJsBytes(audit STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(({wastedBytes}) => wastedBytes); +} catch (e) { + return []; +} +'''; + +SELECT + IF(unminified_js_kbytes <= 200, CEIL(unminified_js_kbytes / 10) * 10, 200) AS unminified_js_kbytes, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER () AS total, + COUNT(0) / SUM(COUNT(0)) OVER () AS pct +FROM ( + SELECT + test.url AS page, + SUM(IFNULL(unminified_js_bytes, 0)) / 1024 AS unminified_js_kbytes + FROM + `httparchive.lighthouse.2021_07_01_mobile` AS test + LEFT JOIN + UNNEST(getUnminifiedJsBytes(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified_js_bytes + GROUP BY + page) +GROUP BY + unminified_js_kbytes +ORDER BY + unminified_js_kbytes diff --git a/sql/2021/javascript/most_requested_content_type.sql b/sql/2021/javascript/most_requested_content_type.sql new file mode 100644 index 00000000000..5561b338b24 --- /dev/null +++ b/sql/2021/javascript/most_requested_content_type.sql @@ -0,0 +1,13 @@ +SELECT + resp_content_type, + COUNT(0) AS count, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM + `httparchive.almanac.requests` +WHERE + date = '2021-07-01' +GROUP BY + client +ORDER BY + client DESC diff --git a/sql/2021/javascript/percentage_usage_of_different_ajax_apis.sql b/sql/2021/javascript/percentage_usage_of_different_ajax_apis.sql new file mode 100644 index 00000000000..db4e8c65adc --- /dev/null +++ b/sql/2021/javascript/percentage_usage_of_different_ajax_apis.sql @@ -0,0 +1,30 @@ +SELECT + client, + percentile, + APPROX_QUANTILES(xmlhttprequest_total, 1000)[ + OFFSET(percentile * 10)] AS xmlhttprequest_total, + APPROX_QUANTILES(fetch_total, 1000)[ + OFFSET(percentile * 10)] AS fetch_total, + APPROX_QUANTILES(beacon_total, 1000)[ + OFFSET(percentile * 10)] AS beacon_total +FROM ( + SELECT + _TABLE_SUFFIX AS client, + JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.ajax_requests.xmlhttprequest') AS xmlhttprequest_total, + JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.ajax_requests.fetch') AS fetch_total, + JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.ajax_requests.beacon') AS beacon_total + FROM + `httparchive.pages.2021_07_01_*`), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/requests_2019.sql b/sql/2021/javascript/requests_2019.sql new file mode 100644 index 00000000000..f9947aece20 --- /dev/null +++ b/sql/2021/javascript/requests_2019.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2019) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2019_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/requests_2020.sql b/sql/2021/javascript/requests_2020.sql new file mode 100644 index 00000000000..087b21ae176 --- /dev/null +++ b/sql/2021/javascript/requests_2020.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2020) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2020_08_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/requests_2021.sql b/sql/2021/javascript/requests_2021.sql new file mode 100644 index 00000000000..fef2951e58d --- /dev/null +++ b/sql/2021/javascript/requests_2021.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2021) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2021_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/resource-hints-prefetch-preload-percentage.sql b/sql/2021/javascript/resource-hints-prefetch-preload-percentage.sql new file mode 100644 index 00000000000..ca9c29468e5 --- /dev/null +++ b/sql/2021/javascript/resource-hints-prefetch-preload-percentage.sql @@ -0,0 +1,49 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + client, + COUNT(DISTINCT IF(prefetch_hint, page, NULL)) AS prefetch_pages, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(prefetch_hint, page, NULL)) / COUNT(DISTINCT page) AS prefetch_pct, + COUNT(DISTINCT IF(preload_hint, page, NULL)) AS preload_pages, + COUNT(DISTINCT IF(preload_hint, page, NULL)) / COUNT(DISTINCT page) AS preload_pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + hint.name = 'prefetch' AND hint.value = 'script' AS prefetch_hint, + hint.name = 'preload' AND hint.value = 'script' AS preload_hint + FROM + `httparchive.pages.2021_07_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint) +GROUP BY + client diff --git a/sql/2021/javascript/resource-hints-preload-prefetch-distribution.sql b/sql/2021/javascript/resource-hints-preload-prefetch-distribution.sql new file mode 100644 index 00000000000..c3ba28337da --- /dev/null +++ b/sql/2021/javascript/resource-hints-preload-prefetch-distribution.sql @@ -0,0 +1,57 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + percentile, + client, + APPROX_QUANTILES(prefetch_hint, 1000)[OFFSET(percentile * 10)] AS prefetch_hints_per_page, + APPROX_QUANTILES(IF(prefetch_hint = 0, NULL, prefetch_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS prefetch_hints_per_page_with_hints, + APPROX_QUANTILES(preload_hint, 1000)[OFFSET(percentile * 10)] AS preload_hints_per_page, + APPROX_QUANTILES(IF(preload_hint = 0, NULL, preload_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS preload_hints_per_page_with_hints +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + COUNTIF(hint.name = 'prefetch' AND hint.value = 'script') AS prefetch_hint, + COUNTIF(hint.name = 'preload' AND hint.value = 'script') AS preload_hint + FROM + `httparchive.pages.2021_07_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint + GROUP BY + client, + page), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/resource_hints.sql b/sql/2021/javascript/resource_hints.sql new file mode 100644 index 00000000000..10488f3d410 --- /dev/null +++ b/sql/2021/javascript/resource_hints.sql @@ -0,0 +1,46 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + client, + COUNT(DISTINCT IF(script_hint, page, NULL)) AS pages, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(script_hint, page, NULL)) / COUNT(DISTINCT page) AS pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + hint.name IN ('prefetch', 'preload') AND hint.value = 'script' AS script_hint + FROM + `httparchive.pages.2021_07_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint) +GROUP BY + client diff --git a/sql/2021/javascript/resource_hints_per_page.sql b/sql/2021/javascript/resource_hints_per_page.sql new file mode 100644 index 00000000000..6fa1cec60cf --- /dev/null +++ b/sql/2021/javascript/resource_hints_per_page.sql @@ -0,0 +1,54 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + percentile, + client, + APPROX_QUANTILES(script_hint, 1000)[OFFSET(percentile * 10)] AS hints_per_page, + APPROX_QUANTILES(IF(script_hint = 0, NULL, script_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS hints_per_page_with_hints +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + COUNTIF(hint.name IN ('prefetch', 'preload') AND hint.value = 'script') AS script_hint + FROM + `httparchive.pages.2021_07_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint + GROUP BY + client, + page), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client diff --git a/sql/2021/javascript/sourcemap_header.sql b/sql/2021/javascript/sourcemap_header.sql new file mode 100644 index 00000000000..2788df59861 --- /dev/null +++ b/sql/2021/javascript/sourcemap_header.sql @@ -0,0 +1,48 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + has_sourcemap_header, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + has_sourcemap_header +ORDER BY + client, + has_sourcemap_header diff --git a/sql/2021/javascript/sourcemap_header_by_3p.sql b/sql/2021/javascript/sourcemap_header_by_3p.sql new file mode 100644 index 00000000000..c863a5371a1 --- /dev/null +++ b/sql/2021/javascript/sourcemap_header_by_3p.sql @@ -0,0 +1,54 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + has_sourcemap_header, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header + FROM + `httparchive.almanac.requests` + WHERE + date = '2021-07-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2021_07_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + has_sourcemap_header +ORDER BY + client, + host, + has_sourcemap_header diff --git a/sql/2021/javascript/unused_js_bytes_distribution.sql b/sql/2021/javascript/unused_js_bytes_distribution.sql new file mode 100644 index 00000000000..62872729613 --- /dev/null +++ b/sql/2021/javascript/unused_js_bytes_distribution.sql @@ -0,0 +1,12 @@ +#standardSQL +# Distribution of unused JS request bytes per page +SELECT + percentile, + APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(report, "$.audits.unused-javascript.details.overallSavingsBytes") AS INT64) / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.lighthouse.2021_07_01_mobile`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile +ORDER BY + percentile diff --git a/sql/2021/javascript/web_components_specs.sql b/sql/2021/javascript/web_components_specs.sql new file mode 100644 index 00000000000..383d0db5503 --- /dev/null +++ b/sql/2021/javascript/web_components_specs.sql @@ -0,0 +1,30 @@ +SELECT + client, + percentile, + APPROX_QUANTILES(custom_elements, 1000)[ + OFFSET(percentile * 10)] AS custom_elements, + APPROX_QUANTILES(shadow_roots, 1000)[ + OFFSET(percentile * 10)] AS shadow_roots, + APPROX_QUANTILES(template, 1000)[ + OFFSET(percentile * 10)] AS template +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.web_component_specs.custom_elements.length') AS INT64) AS custom_elements, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.web_component_specs.shadow_roots.length') AS INT64) AS shadow_roots, + CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, + '$._javascript'), + '$.web_component_specs.template.length') AS INT64) AS template + FROM + `httparchive.pages.2021_07_01_*` ), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client