JavaScript 2021 queries

sql/2021/javascript/ajax_request_per_page.sql

@@ -0,0 +1,22 @@
+SELECT
+  client,
+  percentile,
+  APPROX_QUANTILES(ajax_requests_total, 1000)[
+    OFFSET(percentile * 10)] AS ajax_requests_total
+FROM (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload,
+          '$._javascript'),
+        '$.ajax_requests.total') AS INT64) AS ajax_requests_total
+  FROM
+    `httparchive.pages.2021_07_01_*` ),
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+WHERE
+  ajax_requests_total > 0
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  percentile,
+  client

sql/2021/javascript/bytes_2019.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2019)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2019_07_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  percentile,
+  client

sql/2021/javascript/bytes_2020.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2020)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2020_08_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  percentile,
+  client

sql/2021/javascript/bytes_2021.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2020)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2021_07_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  percentile,
+  client

sql/2021/javascript/bytes_by_3p.sql

@@ -0,0 +1,33 @@
+#standardSQL
+# Distribution of 1P/3P JS bytes
+SELECT
+  percentile,
+  client,
+  host,
+  APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    SUM(respSize) / 1024 AS kbytes
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2021-07-01' AND
+    type = 'script'
+  GROUP BY
+    client,
+    page,
+    host),
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  host
+ORDER BY
+  percentile,
+  client,
+  host

sql/2021/javascript/compression_method.sql

@@ -0,0 +1,47 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  compression,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests
+FROM (
+  SELECT
+    client,
+    page,
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2021-07-01' AND
+    type = 'script')
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  compression
+ORDER BY
+  pct_js_requests DESC

sql/2021/javascript/compression_method_by_3p.sql

@@ -0,0 +1,52 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  host,
+  compression,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2021-07-01' AND
+    type = 'script')
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  host,
+  compression
+ORDER BY
+  pct DESC

sql/2021/javascript/compression_none_by_bytes.sql

@@ -0,0 +1,55 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  host,
+  IF(kbytes < 100, FLOOR(kbytes / 5) * 5, 100) AS kbytes,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2021-07-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    respSize / 1024 AS kbytes
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2021-07-01' AND
+    type = 'script' AND
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') IS NULL)
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  host,
+  kbytes
+ORDER BY
+  client,
+  host,
+  kbytes

sql/2021/javascript/frameworks-bytes-by-framework.sql

@@ -0,0 +1,35 @@
+#standardSQL
+# Sum of JS request bytes per page by framework (2020)
+SELECT
+  percentile,
+  client,
+  app AS js_framework,
+  COUNT(DISTINCT page) AS pages,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    url AS page,
+    bytesJs
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`)
+JOIN (
+  SELECT DISTINCT
+    _TABLE_SUFFIX AS client,
+    url AS page,
+    app
+  FROM
+    `httparchive.technologies.2021_07_01_*`
+  WHERE
+    category = 'JavaScript frameworks')
+USING
+  (client, page),
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  js_framework
+ORDER BY
+  percentile,
+  client,
+  pages DESC

sql/2021/javascript/frameworks_libraries.sql

@@ -0,0 +1,30 @@
+#standardSQL
+# Top JS frameworks and libraries
+SELECT
+  _TABLE_SUFFIX AS client,
+  category,
+  app,
+  COUNT(DISTINCT url) AS pages,
+  total,
+  COUNT(DISTINCT url) / total AS pct
+FROM
+  `httparchive.technologies.2021_07_01_*`
+JOIN (
+  SELECT
+    _TABLE_SUFFIX,
+    COUNT(0) AS total
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+  GROUP BY
+    _TABLE_SUFFIX)
+USING
+  (_TABLE_SUFFIX)
+WHERE
+  category IN ('JavaScript frameworks', 'JavaScript libraries')
+GROUP BY
+  client,
+  category,
+  app,
+  total
+ORDER BY
+  pct DESC

sql/2021/javascript/frameworks_libraries_by_version.sql

@@ -0,0 +1,32 @@
+#standardSQL
+# Top JS frameworks and libraries by version
+SELECT
+  _TABLE_SUFFIX AS client,
+  category,
+  app,
+  info AS version,
+  COUNT(DISTINCT url) AS pages,
+  total,
+  COUNT(DISTINCT url) / total AS pct
+FROM
+  `httparchive.technologies.2021_07_01_*`
+JOIN (
+  SELECT
+    _TABLE_SUFFIX,
+    COUNT(0) AS total
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+  GROUP BY
+    _TABLE_SUFFIX)
+USING
+  (_TABLE_SUFFIX)
+WHERE
+  app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React')
+GROUP BY
+  client,
+  category,
+  app,
+  info,
+  total
+ORDER BY
+  pct DESC