Third Party 2021 Queries

sql/2021/third-parties/README.md

@@ -0,0 +1,18 @@
+# 2021 Third-Party queries
+
+<!--
+  This directory contains all of the 2021 Third-Party chapter queries.
+
+  Each query should have a corresponding `metric_name.sql` file.
+  Note that readers are linked to this directory, so try to make the SQL file names descriptive for easy browsing.
+
+  Analysts: if helpful, you can use this README to give additional info about the queries.
+-->
+
+Resources:
+
+- [Chapter issue](https://github.com/HTTPArchive/almanac.httparchive.org/issues/2145)
+- [Planning doc](https://docs.google.com/document/d/164HhV76iVT2qVfFY2kzyr44eIcGAw3fWqNTbuoRfRvE/edit?usp=sharing)
+- [Results sheet](https://docs.google.com/spreadsheets/d/1tf4RMF8SYr6he9tbqt61yuFJ_QK-F-i7XPxaPkpKSDI/edit?usp=sharing/)
+- [2019 chapter](https://almanac.httparchive.org/en/2019/third-parties)
+- [2020 chapter](https://almanac.httparchive.org/en/2020/third-parties)

sql/2021/third-parties/distribution_of_3XX_response_body_size.sql

@@ -0,0 +1,53 @@
+#standardSQL
+# Distribution of response body size by redirected third parties
+# HTTP status codes documentation: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    url,
+    status,
+    respBodySize AS body_size
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+),
+
+base AS (
+  SELECT
+    client,
+    domain,
+    IF(status BETWEEN 300 AND 399, 1, 0) AS redirected,
+    body_size
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+)
+
+SELECT
+  client,
+  percentile,
+  APPROX_QUANTILES(body_size, 1000)[OFFSET(percentile * 10)] AS approx_redirect_body_size
+FROM
+  base,
+  UNNEST(GENERATE_ARRAY(1, 100)) AS percentile
+WHERE
+  redirected = 1
+GROUP BY
+  client,
+  percentile
+ORDER BY
+  client,
+  percentile

sql/2021/third-parties/distribution_of_size_and_time_by_third_parties.sql

@@ -0,0 +1,55 @@
+#standardSQL
+# Distribution of third party requests size and time by category
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    url,
+    respBodySize AS body_size,
+    time
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    category,
+    domain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+),
+
+base AS (
+  SELECT
+    client,
+    category,
+    body_size,
+    time
+  FROM
+    requests
+  INNER JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+)
+
+SELECT
+  client,
+  category,
+  percentile,
+  APPROX_QUANTILES(body_size, 1000)[OFFSET(percentile * 10)] AS body_size,
+  APPROX_QUANTILES(time, 1000)[OFFSET(percentile * 10)] AS time
+FROM
+  base,
+  UNNEST(GENERATE_ARRAY(1, 100)) AS percentile
+GROUP BY
+  client,
+  category,
+  percentile
+ORDER BY
+  client,
+  category,
+  percentile

sql/2021/third-parties/distribution_of_third_parties_by_number_of_websites.sql

@@ -0,0 +1,54 @@
+#standardSQL
+# Distribution of third parties by number of websites
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    url
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain,
+    canonicalDomain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+),
+
+base AS (
+  SELECT
+    client,
+    canonicalDomain,
+    COUNT(DISTINCT page) AS pages_per_third_party
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+  WHERE
+    canonicalDomain IS NOT NULL
+  GROUP BY
+    client,
+    canonicalDomain
+)
+
+SELECT
+  client,
+  percentile,
+  APPROX_QUANTILES(pages_per_third_party, 1000)[OFFSET(percentile * 10)] AS approx_pages_per_third_party
+FROM
+  base,
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  client,
+  percentile
+ORDER BY
+  client,
+  percentile

sql/2021/third-parties/distribution_of_websites_by number_of_third_parties.sql

@@ -0,0 +1,51 @@
+#standardSQL
+# Distribution of websites by number of third party
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    url
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+),
+
+base AS (
+  SELECT
+    client,
+    page,
+    COUNT(domain) AS third_parties_per_page
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+  GROUP BY
+    client,
+    page
+)
+
+SELECT
+  client,
+  percentile,
+  APPROX_QUANTILES(third_parties_per_page, 1000)[OFFSET(percentile * 10)] AS approx_third_parties_per_page
+FROM
+  base,
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  client,
+  percentile
+ORDER BY
+  client,
+  percentile

sql/2021/third-parties/percent_of_third_parties_by_content_type.sql

@@ -0,0 +1,42 @@
+#standardSQL
+# Percent of third party requests by content type.
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    url,
+    type AS contentType
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+)
+
+SELECT
+  client,
+  contentType,
+  COUNT(0) AS requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total_requests,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_requests
+FROM
+  requests
+LEFT JOIN
+  third_party
+ON
+  NET.HOST(requests.url) = NET.HOST(third_party.domain)
+WHERE
+  domain IS NOT NULL
+GROUP BY
+  client,
+  contentType
+ORDER BY
+  client,
+  contentType

sql/2021/third-parties/percent_of_third_party_cache.sql

@@ -0,0 +1,63 @@
+#standardSQL
+# Percent of third party requests cached
+# Cache-Control documentation: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#Directives
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    resp_cache_control,
+    status,
+    respOtherHeaders,
+    reqOtherHeaders,
+    type,
+    url
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain
+  FROM
+    `httparchive.almanac.third_parties`
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+),
+
+base AS (
+  SELECT
+    client,
+    type,
+    IF(
+      (
+        status IN (301, 302, 307, 308, 410) AND
+        NOT REGEXP_CONTAINS(resp_cache_control, r'(?i)private|no-store') AND
+        NOT REGEXP_CONTAINS(reqOtherHeaders, r'Authorization')
+      ) OR
+      (
+        status IN (301, 302, 307, 308, 410) OR
+        REGEXP_CONTAINS(resp_cache_control, r'public|max-age|s-maxage') OR
+        REGEXP_CONTAINS(respOtherHeaders, r'Expires')
+      ), 1, 0) AS cached
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+  WHERE
+    domain IS NOT NULL
+)
+
+SELECT
+  client,
+  type,
+  SUM(cached) AS cached_requests,
+  COUNT(0) AS total_requests,
+  SUM(cached) / COUNT(0) AS pct_cached_requests
+FROM
+  base
+GROUP BY
+  client,
+  type