Third party 2021 query amends (#2404)

* Add limit to 50 * Linting fixes * More ranking queries * Fix blocking query and linting * Update sql/2021/third-parties/third_parties_blocking_main_thread.sql * Update sql/2021/third-parties/third_parties_blocking_main_thread.sql Co-authored-by: Rick Viscomi <[email protected]>
HTTPArchive · Oct 25, 2021 · 33ad00f · 33ad00f
1 parent e80bb31
commit 33ad00f
Show file tree

Hide file tree

Showing 20 changed files with 371 additions and 54 deletions.
diff --git a/sql/2021/third-parties/distribution_of_3XX_response_body_size.sql b/sql/2021/third-parties/distribution_of_3XX_response_body_size.sql
@@ -5,6 +5,7 @@
 WITH requests AS (
   SELECT
     _TABLE_SUFFIX AS client,
+    pageid AS page,
     url,
     status,
     respBodySize AS body_size
@@ -14,12 +15,22 @@ WITH requests AS (
 
 third_party AS (
   SELECT
-    domain
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (

diff --git a/sql/2021/third-parties/distribution_of_size_and_time_by_third_parties.sql b/sql/2021/third-parties/distribution_of_size_and_time_by_third_parties.sql
@@ -4,6 +4,7 @@
 WITH requests AS (
   SELECT
     _TABLE_SUFFIX AS client,
+    pageid AS page,
     url,
     respBodySize AS body_size,
     time
@@ -13,13 +14,22 @@ WITH requests AS (
 
 third_party AS (
   SELECT
+    domain,
     category,
-    domain
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (

diff --git a/sql/2021/third-parties/distribution_of_third_parties_by_number_of_websites.sql b/sql/2021/third-parties/distribution_of_third_parties_by_number_of_websites.sql
@@ -13,12 +13,23 @@ WITH requests AS (
 third_party AS (
   SELECT
     domain,
-    canonicalDomain
+    canonicalDomain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    canonicalDomain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (

diff --git a/...f_websites_by number_of_third_parties.sql → ...f_websites_by_number_of_third_parties.sql b/...f_websites_by number_of_third_parties.sql → ...f_websites_by_number_of_third_parties.sql
@@ -12,12 +12,22 @@ WITH requests AS (
 
 third_party AS (
   SELECT
-    domain
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (

diff --git a/sql/2021/third-parties/number_of_third_parties_by_rank.sql b/sql/2021/third-parties/number_of_third_parties_by_rank.sql
@@ -0,0 +1,77 @@
+#standardSQL
+# Number of third-parties per websites by rank
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    url
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+pages AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    rank
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
+  FROM
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
+  WHERE
+    date = '2021-07-01' AND
+    category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
+),
+
+base AS (
+  SELECT
+    client,
+    page,
+    rank,
+    COUNT(domain) AS third_parties_per_page
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+  INNER JOIN
+    pages
+  USING
+    (client, page)
+  GROUP BY
+    client,
+    page,
+    rank
+)
+
+SELECT
+  client,
+  rank_grouping,
+  APPROX_QUANTILES(third_parties_per_page, 1000)[OFFSET(500)] AS p50_third_parties_per_page
+FROM
+  base,
+  UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping
+WHERE
+  rank <= rank_grouping
+GROUP BY
+  client,
+  rank_grouping
+ORDER BY
+  client,
+  rank_grouping
diff --git a/sql/2021/third-parties/number_of_third_parties_by_rank_and_category.sql b/sql/2021/third-parties/number_of_third_parties_by_rank_and_category.sql
@@ -0,0 +1,83 @@
+#standardSQL
+# Number of third-parties per websites by rank and category
+
+WITH requests AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    url
+  FROM
+    `httparchive.summary_requests.2021_07_01_*`
+),
+
+pages AS (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    pageid AS page,
+    rank
+  FROM
+    `httparchive.summary_pages.2021_07_01_*`
+),
+
+third_party AS (
+  SELECT
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
+  FROM
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
+  WHERE
+    date = '2021-07-01' AND
+    category NOT IN ('hosting')
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
+),
+
+base AS (
+  SELECT
+    client,
+    category,
+    page,
+    rank,
+    COUNT(domain) AS third_parties_per_page
+  FROM
+    requests
+  LEFT JOIN
+    third_party
+  ON
+    NET.HOST(requests.url) = NET.HOST(third_party.domain)
+  INNER JOIN
+    pages
+  USING
+    (client, page)
+  GROUP BY
+    client,
+    category,
+    page,
+    rank
+)
+
+SELECT
+  client,
+  category,
+  rank_grouping,
+  APPROX_QUANTILES(third_parties_per_page, 1000)[OFFSET(500)] AS p50_third_parties_per_page
+FROM
+  base,
+  UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping
+WHERE
+  rank <= rank_grouping
+GROUP BY
+  client,
+  category,
+  rank_grouping
+ORDER BY
+  client,
+  category,
+  rank_grouping
diff --git a/sql/2021/third-parties/percent_of_third_parties_by_content_type.sql b/sql/2021/third-parties/percent_of_third_parties_by_content_type.sql
@@ -4,6 +4,7 @@
 WITH requests AS (
   SELECT
     _TABLE_SUFFIX AS client,
+    pageid AS page,
     url,
     type AS contentType
   FROM
@@ -12,12 +13,22 @@ WITH requests AS (
 
 third_party AS (
   SELECT
-    domain
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 )
 
 SELECT

diff --git a/sql/2021/third-parties/percent_of_third_party_cache.sql b/sql/2021/third-parties/percent_of_third_party_cache.sql
@@ -10,19 +10,30 @@ WITH requests AS (
     respOtherHeaders,
     reqOtherHeaders,
     type,
-    url
+    url,
+    pageid AS page
   FROM
     `httparchive.summary_requests.2021_07_01_*`
 ),
 
 third_party AS (
   SELECT
-    domain
+    domain,
+    category,
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (

diff --git a/sql/2021/third-parties/percent_of_third_party_loaded_before_DOMContentLoaded.sql b/sql/2021/third-parties/percent_of_third_party_loaded_before_DOMContentLoaded.sql
@@ -22,13 +22,22 @@ pages AS (
 
 third_party AS (
   SELECT
+    domain,
     category,
-    domain
+    COUNT(DISTINCT page) AS page_usage
   FROM
-    `httparchive.almanac.third_parties`
+    `httparchive.almanac.third_parties` tp
+  JOIN
+    requests r
+  ON NET.HOST(r.url) = NET.HOST(tp.domain)
   WHERE
     date = '2021-07-01' AND
     category != 'hosting'
+  GROUP BY
+    domain,
+    category
+  HAVING
+    page_usage >= 50
 ),
 
 base AS (