diff --git a/sql/2020/01_CSS/custom_property_cycles_max.sql b/sql/2020/01_CSS/custom_property_cycles_max.sql new file mode 100644 index 00000000000..e0db0077971 --- /dev/null +++ b/sql/2020/01_CSS/custom_property_cycles_max.sql @@ -0,0 +1,106 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getCustomPropertyMaxCycles(payload STRING) RETURNS INT64 LANGUAGE js AS ''' +try { + function compute(vars) { + function walkElements(node, callback, parent) { + if (Array.isArray(node)) { + for (let n of node) { + walkElements(n, callback, parent); + } + } + else { + callback(node, parent); + + if (node.children) { + walkElements(node.children, callback, node); + } + } + } + + let ret = { + max_length: 0, + cycles_or_initial: 0 + }; + + function countDependencyLength(node, property) { + if (!node) { + return 0; + } + + let declarations = node.declarations; + + if (!declarations || !(property in declarations)) { + return countDependencyLength(node.parent, property); + } + + let o = declarations[property]; + + if (!o.references || o.references.length === 0) { + return 0; + } + + let lengths = o.references.map(p => countDependencyLength(node, p)); + + return 1 + Math.max(...lengths); + } + + walkElements(vars.computed, (node, parent) => { + if (parent && !node.parent) { + node.parent = parent; + } + + if (node.declarations) { + for (let property in node.declarations) { + + let o = node.declarations[property]; + if (o.computed && o.computed.trim() !== o.value.trim() && (o.computed === "initial" || o.computed === "null")) { + // Cycle or missing ref + ret.cycles_or_initial++; + } + else { + let depth = countDependencyLength(node, property); + + if (depth > ret.max_length) { + ret.max_length = depth; + } + + incrementByKey(ret, depth); + } + } + } + }); + + return ret; + } + var $ = JSON.parse(payload); + var vars = JSON.parse($['_css-variables']); + var custom_props = compute(vars); + if (!('max_length' in custom_props)) return null; + return custom_props.max_length; +} catch (e) { + return null; +} +''' +OPTIONS (library="gs://httparchive/lib/css-utils.js"); + +SELECT + percentile, + client, + APPROX_QUANTILES(max_cycles, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS max_cycles_per_page +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url, + MAX(getCustomPropertyMaxCycles(payload)) AS max_cycles + FROM + `httparchive.pages.2020_08_01_*` + GROUP BY + client, + url), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + percentile, + client \ No newline at end of file diff --git a/sql/2020/01_CSS/custom_propery_depth.sql b/sql/2020/01_CSS/custom_property_depth.sql similarity index 100% rename from sql/2020/01_CSS/custom_propery_depth.sql rename to sql/2020/01_CSS/custom_property_depth.sql diff --git a/sql/2020/01_CSS/custom_property_registered.sql b/sql/2020/01_CSS/custom_property_registered.sql index e9b4b71543c..0edb40aa8aa 100644 --- a/sql/2020/01_CSS/custom_property_registered.sql +++ b/sql/2020/01_CSS/custom_property_registered.sql @@ -40,17 +40,14 @@ try { } '''; -SELECT +SELECT DISTINCT _TABLE_SUFFIX AS client, prop, - COUNT(0) AS pages, - SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, - COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct + COUNT(DISTINCT url) OVER (PARTITION BY _TABLE_SUFFIX, prop) AS pages, + COUNT(DISTINCT url) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNT(DISTINCT url) OVER (PARTITION BY _TABLE_SUFFIX, prop) / COUNT(DISTINCT url) OVER (PARTITION BY _TABLE_SUFFIX) AS pct FROM `httparchive.pages.2020_08_01_*`, UNNEST(getCustomPropertiesWithComputedStyle(payload)) AS prop -GROUP BY - client, - prop ORDER BY pct DESC \ No newline at end of file diff --git a/sql/2020/01_CSS/custom_property_root.sql b/sql/2020/01_CSS/custom_property_root.sql index f4c1a860f60..7035da15657 100644 --- a/sql/2020/01_CSS/custom_property_root.sql +++ b/sql/2020/01_CSS/custom_property_root.sql @@ -63,9 +63,9 @@ SELECT SUM(freq) AS freq, SUM(SUM(freq)) OVER (PARTITION BY client) AS total, SUM(freq) / SUM(SUM(freq)) OVER (PARTITION BY client) AS pct, - COUNT(DISTINCT page) AS pages, + COUNT(DISTINCT IF(freq > 0, page, NULL)) AS pages, total_pages, - COUNT(DISTINCT page) / total_pages AS pct_pages + COUNT(DISTINCT IF(freq > 0, page, NULL)) / total_pages AS pct_pages FROM ( SELECT _TABLE_SUFFIX AS client, diff --git a/sql/2020/01_CSS/gradient_bg_properties.sql b/sql/2020/01_CSS/gradient_bg_properties.sql index fd9eaa91e30..a5eb94311b8 100644 --- a/sql/2020/01_CSS/gradient_bg_properties.sql +++ b/sql/2020/01_CSS/gradient_bg_properties.sql @@ -143,7 +143,7 @@ OPTIONS (library="gs://httparchive/lib/css-utils.js"); SELECT client, - IF(REGEXP_CONTAINS(property, r'(?i)^(-webkit-|-o-|-moz-|-ms-)?background(-image)?'), 'bg', 'other') AS property_type, + IF(REGEXP_CONTAINS(property, r'(?i)(\b|_)background\b'), 'bg', 'other') AS property_type, COUNT(DISTINCT page) AS pages, total, COUNT(DISTINCT page) / total AS pct diff --git a/sql/2020/01_CSS/gradient_stops_per_stylesheet.sql b/sql/2020/01_CSS/gradient_stops_per_gradient.sql similarity index 98% rename from sql/2020/01_CSS/gradient_stops_per_stylesheet.sql rename to sql/2020/01_CSS/gradient_stops_per_gradient.sql index 0bea0f9e6b4..a508f128648 100644 --- a/sql/2020/01_CSS/gradient_stops_per_stylesheet.sql +++ b/sql/2020/01_CSS/gradient_stops_per_gradient.sql @@ -157,7 +157,7 @@ OPTIONS (library="gs://httparchive/lib/css-utils.js"); SELECT percentile, client, - APPROX_QUANTILES(color_stops, 1000 IGNORE NULLS)[OFFSET(500)] AS median_color_stop + APPROX_QUANTILES(color_stops, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS color_stops_per_gradient FROM `httparchive.almanac.parsed_css`, UNNEST(getColorStops(css)) AS color_stops, diff --git a/sql/2020/01_CSS/houdini_animated_custom_properties.sql b/sql/2020/01_CSS/houdini_animated_custom_properties.sql index e9b1372a7a7..f4afde31343 100644 --- a/sql/2020/01_CSS/houdini_animated_custom_properties.sql +++ b/sql/2020/01_CSS/houdini_animated_custom_properties.sql @@ -22,20 +22,19 @@ try { ''' OPTIONS (library="gs://httparchive/lib/css-utils.js"); -SELECT +SELECT DISTINCT client, custom_property, - COUNT(DISTINCT url) AS pages, - COUNT(0) AS freq, - SUM(COUNT(0)) OVER (PARTITION BY client) AS total, - COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct + COUNT(DISTINCT page) OVER (PARTITION BY client, custom_property) AS pages, + COUNT(DISTINCT page) OVER (PARTITION BY client) AS total_pages, + COUNT(DISTINCT page) OVER (PARTITION BY client, custom_property) / COUNT(DISTINCT page) OVER (PARTITION BY client) AS pct_pages, + COUNT(0) OVER (PARTITION BY client, custom_property) AS freq, + COUNT(0) OVER (PARTITION BY client) AS total, + COUNT(0) OVER (PARTITION BY client, custom_property) / COUNT(0) OVER (PARTITION BY client) AS pct FROM `httparchive.almanac.parsed_css`, UNNEST(getAnimatedCustomProperties(css)) AS custom_property WHERE date = '2020-08-01' -GROUP BY - client, - custom_property ORDER BY pct DESC \ No newline at end of file diff --git a/sql/2020/01_CSS/meta_unknown_properties.sql b/sql/2020/01_CSS/meta_unknown_properties.sql index 6035ecea8a5..077cc3148bd 100644 --- a/sql/2020/01_CSS/meta_unknown_properties.sql +++ b/sql/2020/01_CSS/meta_unknown_properties.sql @@ -36,13 +36,15 @@ OPTIONS (library="gs://httparchive/lib/css-utils.js"); SELECT * FROM ( - SELECT + SELECT DISTINCT client, property, - COUNT(DISTINCT page) AS pages, - SUM(freq) AS freq, - SUM(SUM(freq)) OVER (PARTITION BY client) AS total, - SUM(freq) / SUM(SUM(freq)) OVER (PARTITION BY client) AS pct + COUNT(DISTINCT page) OVER (PARTITION BY client, property) AS pages, + COUNT(DISTINCT page) OVER (PARTITION BY client) AS total, + COUNT(DISTINCT page) OVER (PARTITION BY client, property) / COUNT(DISTINCT page) OVER (PARTITION BY client) AS pct_pages, + SUM(freq) OVER (PARTITION BY client, property) AS freq, + SUM(freq) OVER (PARTITION BY client) AS total, + SUM(freq) OVER (PARTITION BY client, property) / SUM(freq) OVER (PARTITION BY client) AS pct FROM ( SELECT client, @@ -54,10 +56,7 @@ FROM ( UNNEST(getUnknownProperties(css)) AS property WHERE date = '2020-08-01' AND - LENGTH(property) > 1) - GROUP BY - client, - property) + LENGTH(property.property) > 1)) WHERE pct >= 0.01 ORDER BY diff --git a/sql/2020/01_CSS/vendor_prefix_keywords.sql b/sql/2020/01_CSS/vendor_prefix_keywords.sql index 07dbfaff324..542b6e127f6 100644 --- a/sql/2020/01_CSS/vendor_prefix_keywords.sql +++ b/sql/2020/01_CSS/vendor_prefix_keywords.sql @@ -35,8 +35,10 @@ try { } // Prefixed keywords - for (let k of value.matchAll(/(?