CSS 2020 queries (#1577)

* css feedback

* rework

* custom props and units

* note

* Add queries for all functions and all properties

* Fix bug with all properties query

* Normalize non-custom properties to lowercase

* Add query for all Blink Features

For @supports and a number of other things

* Create keyword_totals.sql

* Create vendor_prefix_summary.sql

* Create meta_shorthand_first_pages.sql

* spaces for tabs

* Update sql/2020/01_CSS/vendor_prefix_summary.sql

Co-authored-by: Rick Viscomi <[email protected]>

* Update sql/2020/01_CSS/vendor_prefix_summary.sql

Co-authored-by: Rick Viscomi <[email protected]>

* Update meta_longhand_first_distribution.sql

* Avoid counting vendor prefix functions and keywords when value.length > 1000

* Create repetition.sql

* Update repetition.sql

* Improve repetition query

* Update vendor_prefix_summary.sql

* Update sass_nesting.sql

* Update all_features.sql

Query the summary table instead

* Update sql/2020/01_CSS/vendor_prefix_summary.sql

* support AST >= 100KB for color/gradient queries

* gradient adoption

* gradient hints

Co-authored-by: Rick Viscomi <[email protected]>
Co-authored-by: Rick Viscomi <[email protected]>

sql/2020/01_CSS/all_features.sql

@@ -0,0 +1,12 @@
+SELECT
+  client,
+  feature,
+  num_urls AS freq,
+  total_urls AS total,
+  pct_urls AS pct_pages
+FROM
+  `httparchive.blink_features.usage`
+WHERE
+  yyyymmdd = '20200801'
+ORDER BY
+  pct_pages DESC

sql/2020/01_CSS/all_functions.sql

@@ -0,0 +1,54 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getProperties(css STRING) RETURNS ARRAY<STRING> LANGUAGE js AS '''
+try {
+	function compute() {
+		let ret = {};
+
+		walkDeclarations(ast, ({property, value}) => {
+			if (value.length > 1000 || !value.includes("(") || !value.includes(")")) {
+				return;
+			}
+
+			for (let {name} of extractFunctionCalls(value)) {
+				incrementByKey(ret, name);
+			}
+		});
+
+		return sortObject(ret);
+
+	}
+
+	let ast = JSON.parse(css);
+	let props = compute(ast);
+	return Object.entries(props).flatMap(([prop, freq]) => {
+		return Array(freq).fill(prop);
+	});
+}
+catch (e) {
+	return [];
+}
+'''
+OPTIONS (library="gs://httparchive/lib/css-utils.js");
+
+SELECT
+  *
+FROM (
+  SELECT
+    client,
+    prop,
+    COUNT(DISTINCT page) AS pages,
+    COUNT(0) AS freq,
+    SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+    COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
+  FROM
+    `httparchive.almanac.parsed_css`,
+    UNNEST(getProperties(css)) AS prop
+  WHERE
+    date = '2020-08-01'
+  GROUP BY
+    client,
+    prop)
+WHERE
+  pages >= 1000
+ORDER BY
+  pct DESC

sql/2020/01_CSS/all_properties.sql

@@ -0,0 +1,51 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getProperties(css STRING) RETURNS ARRAY<STRING> LANGUAGE js AS '''
+try {
+	function compute(ast) {
+		let ret = {};
+
+		walkDeclarations(ast, ({property, value}) => {
+			if (!property.startsWith("--")) { // Custom props are case sensitive
+				property = property.toLowerCase();
+			}
+			
+			incrementByKey(ret, property);
+		});
+
+		return sortObject(ret);
+	}
+
+	let ast = JSON.parse(css);
+	let props = compute(ast);
+	return Object.entries(props).flatMap(([prop, freq]) => {
+		return Array(freq).fill(prop);
+	});
+}
+catch (e) {
+	return [];
+}
+'''
+OPTIONS (library="gs://httparchive/lib/css-utils.js");
+
+SELECT
+  *
+FROM (
+  SELECT
+    client,
+    prop,
+    COUNT(DISTINCT page) AS pages,
+    COUNT(0) AS freq,
+    SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+    COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
+  FROM
+    `httparchive.almanac.parsed_css`,
+    UNNEST(getProperties(css)) AS prop
+  WHERE
+    date = '2020-08-01'
+  GROUP BY
+    client,
+    prop)
+WHERE
+  pages >= 1000
+ORDER BY
+  pct DESC

sql/2020/01_CSS/color_arg_comma.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\b/gi);
       usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\b/gi);
       usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\b/gi);
@@ -163,8 +164,6 @@ FROM (
   FROM
     `httparchive.almanac.parsed_css`
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client

sql/2020/01_CSS/color_formats.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       // First remove url() references to avoid them mucking the results
       for (let f of extractFunctionCalls(value, {names: "url"})) {
         let [start, end] = f.pos;
@@ -184,9 +185,7 @@ FROM (
     `httparchive.almanac.parsed_css`,
     UNNEST(getColorFormats(css)) AS format
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client,
   format

sql/2020/01_CSS/color_functions.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\b/gi);
       usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\b/gi);
       usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\b/gi);
@@ -166,8 +167,6 @@ FROM (
     UNNEST(getColorFunctions(css)) AS function
   WHERE
     date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024 AND
     function IS NOT NULL)
 JOIN (
   SELECT

sql/2020/01_CSS/color_keywords.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       // First remove url() references to avoid them mucking the results
       for (let f of extractFunctionCalls(value, {names: "url"})) {
         let [start, end] = f.pos;
@@ -174,9 +175,7 @@ FROM (
     `httparchive.almanac.parsed_css`,
     UNNEST(getColorKeywords(css)) AS keyword
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client,
   keyword

sql/2020/01_CSS/color_keywords_no_system_casefold.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       // First remove url() references to avoid them mucking the results
       for (let f of extractFunctionCalls(value, {names: "url"})) {
         let [start, end] = f.pos;
@@ -174,9 +175,7 @@ FROM (
     `httparchive.almanac.parsed_css`,
     UNNEST(getColorKeywords(css)) AS keyword
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client,
   keyword

sql/2020/01_CSS/color_p3.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       // First remove url() references to avoid them mucking the results
       for (let f of extractFunctionCalls(value, {names: "url"})) {
         let [start, end] = f.pos;
@@ -173,9 +174,7 @@ FROM (
     `httparchive.almanac.parsed_css`,
     UNNEST(getP3Usage(css)) AS p3
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client,
   p3

sql/2020/01_CSS/color_spaces.sql

@@ -89,6 +89,7 @@ try {
     }
 
     walkDeclarations(ast, ({property, value}) => {
+      if (value.length > 1000) return;
       // First remove url() references to avoid them mucking the results
       for (let f of extractFunctionCalls(value, {names: "url"})) {
         let [start, end] = f.pos;
@@ -172,9 +173,7 @@ FROM (
     `httparchive.almanac.parsed_css`,
     UNNEST(getColorSpaces(css)) AS color_space
   WHERE
-    date = '2020-08-01' AND
-    # Limit the size of the CSS to avoid OOM crashes.
-    LENGTH(css) < 0.1 * 1024 * 1024)
+    date = '2020-08-01')
 GROUP BY
   client,
   color_space