From 891abf9bd30e7e270230b743f313bec9f7be8d50 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Thu, 21 Nov 2024 09:58:11 -0500 Subject: [PATCH] protocols/flow: add array inference to protocol Adds "array" information to projection inference, which will include the types of items the array contains, as well as the minimum and maximum number of items it is allowed to contain. This is only adding the (optional) field to the protocol. All users of the updated rust structs will need to be updated to use the new structs before the new field is populated. Things that will need updated to use the updated rust structs include: - `derive-typescript` - Rust connectors: `source-kafka` and `source-http-ingest` - dekaf - flowctl, with gitpod implications - ATF --- crates/assemble/src/lib.rs | 5 +- .../snapshots/assemble__test__inference.snap | 3 + crates/proto-flow/src/flow.rs | 24 + crates/proto-flow/src/flow.serde.rs | 166 ++++ crates/proto-flow/tests/regression.rs | 6 + .../regression__capture_request_json.snap | 27 + .../regression__capture_request_proto.snap | 250 +++--- .../regression__capture_spec_json.snap | 9 + .../regression__capture_spec_proto.snap | 80 +- .../regression__collection_spec_json.snap | 9 + .../regression__collection_spec_proto.snap | 46 +- .../regression__derivation_spec_json.snap | 18 + .../regression__derivation_spec_proto.snap | 157 ++-- .../regression__derive_request_json.snap | 27 + .../regression__derive_request_proto.snap | 203 ++--- ...regression__materialization_spec_json.snap | 9 + ...egression__materialization_spec_proto.snap | 109 +-- .../regression__materialize_request_json.snap | 27 + ...regression__materialize_request_proto.snap | 345 ++++---- ...lection_schema_contains_flow_document.snap | 4 + ...n_is_skipped_when_shards_are_disabled.snap | 15 + ..._tests__disabled_bindings_are_ignored.snap | 24 + .../scenario_tests__golden_all_visits.snap | 149 ++++ ...zation_constraints_on_excluded_fields.snap | 12 + ...tion_not_created_for_empty_properties.snap | 5 + .../snapshots/transition_tests__inserts.snap | 25 + ..._update_collection_becomes_derivation.snap | 5 + .../snapshots/transition_tests__updates.snap | 25 + go/protocols/flow/flow.pb.go | 810 +++++++++++++----- go/protocols/flow/flow.proto | 17 + 30 files changed, 1789 insertions(+), 822 deletions(-) diff --git a/crates/assemble/src/lib.rs b/crates/assemble/src/lib.rs index ab658c5fb7..57e060164e 100644 --- a/crates/assemble/src/lib.rs +++ b/crates/assemble/src/lib.rs @@ -63,6 +63,7 @@ pub fn inference(shape: &Shape, exists: Exists) -> flow::Inference { } else { None }, + array: None, } } @@ -120,7 +121,9 @@ pub fn partition_template( let compression_codec = compression_codec(codec.unwrap_or(models::CompressionCodec::Gzip)); // If an explicit flush interval isn't provided, default to 24 hours - let flush_interval = flush_interval.unwrap_or(std::time::Duration::from_secs(24 * 3600)).into(); + let flush_interval = flush_interval + .unwrap_or(std::time::Duration::from_secs(24 * 3600)) + .into(); // If a fragment length isn't set, default and then map MB to bytes. let length = (length.unwrap_or(512) as i64) << 20; diff --git a/crates/assemble/src/snapshots/assemble__test__inference.snap b/crates/assemble/src/snapshots/assemble__test__inference.snap index 3a7dc9e503..54836fa844 100644 --- a/crates/assemble/src/snapshots/assemble__test__inference.snap +++ b/crates/assemble/src/snapshots/assemble__test__inference.snap @@ -22,6 +22,7 @@ expression: "&[out1, out2, out3]" secret: true, exists: Must, numeric: None, + array: None, }, Inference { types: [ @@ -34,6 +35,7 @@ expression: "&[out1, out2, out3]" secret: true, exists: May, numeric: None, + array: None, }, Inference { types: [ @@ -61,5 +63,6 @@ expression: "&[out1, out2, out3]" maximum: 1000.0, }, ), + array: None, }, ] diff --git a/crates/proto-flow/src/flow.rs b/crates/proto-flow/src/flow.rs index 4211d33f71..d82518e1f7 100644 --- a/crates/proto-flow/src/flow.rs +++ b/crates/proto-flow/src/flow.rs @@ -86,6 +86,8 @@ pub struct Inference { pub exists: i32, #[prost(message, optional, tag = "9")] pub numeric: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub array: ::core::option::Option, } /// Nested message and enum types in `Inference`. pub mod inference { @@ -130,6 +132,28 @@ pub mod inference { #[prost(double, tag = "4")] pub maximum: f64, } + /// Array type-specific inferences. Will be nil if types doesn't include + /// "array", or if the specification was built prior to array inference + /// existing in the protocol. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Array { + /// Minimum number of items the array must contain. + #[prost(uint32, tag = "1")] + pub min_items: u32, + /// True if there is an inferred maximum allowed number of items the array + /// may contain, otherwise False. + #[prost(bool, tag = "2")] + pub has_max_items: bool, + /// Maximum number of items the array may contain. + #[prost(uint32, tag = "3")] + pub max_items: u32, + /// The possible types of items contained in this array. + /// Subset of ["null", "boolean", "object", "array", "integer", "numeric", + /// "string"]. + #[prost(string, repeated, tag = "4")] + pub items: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + } /// Exists enumerates the possible states of existence for a location. #[derive( Clone, diff --git a/crates/proto-flow/src/flow.serde.rs b/crates/proto-flow/src/flow.serde.rs index 174d897227..1166cd7139 100644 --- a/crates/proto-flow/src/flow.serde.rs +++ b/crates/proto-flow/src/flow.serde.rs @@ -2573,6 +2573,9 @@ impl serde::Serialize for Inference { if self.numeric.is_some() { len += 1; } + if self.array.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("flow.Inference", len)?; if !self.types.is_empty() { struct_ser.serialize_field("types", &self.types)?; @@ -2600,6 +2603,9 @@ impl serde::Serialize for Inference { if let Some(v) = self.numeric.as_ref() { struct_ser.serialize_field("numeric", v)?; } + if let Some(v) = self.array.as_ref() { + struct_ser.serialize_field("array", v)?; + } struct_ser.end() } } @@ -2619,6 +2625,7 @@ impl<'de> serde::Deserialize<'de> for Inference { "secret", "exists", "numeric", + "array", ]; #[allow(clippy::enum_variant_names)] @@ -2631,6 +2638,7 @@ impl<'de> serde::Deserialize<'de> for Inference { Secret, Exists, Numeric, + Array, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -2660,6 +2668,7 @@ impl<'de> serde::Deserialize<'de> for Inference { "secret" => Ok(GeneratedField::Secret), "exists" => Ok(GeneratedField::Exists), "numeric" => Ok(GeneratedField::Numeric), + "array" => Ok(GeneratedField::Array), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -2687,6 +2696,7 @@ impl<'de> serde::Deserialize<'de> for Inference { let mut secret__ = None; let mut exists__ = None; let mut numeric__ = None; + let mut array__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Types => { @@ -2737,6 +2747,12 @@ impl<'de> serde::Deserialize<'de> for Inference { } numeric__ = map_.next_value()?; } + GeneratedField::Array => { + if array__.is_some() { + return Err(serde::de::Error::duplicate_field("array")); + } + array__ = map_.next_value()?; + } } } Ok(Inference { @@ -2748,12 +2764,162 @@ impl<'de> serde::Deserialize<'de> for Inference { secret: secret__.unwrap_or_default(), exists: exists__.unwrap_or_default(), numeric: numeric__, + array: array__, }) } } deserializer.deserialize_struct("flow.Inference", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for inference::Array { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.min_items != 0 { + len += 1; + } + if self.has_max_items { + len += 1; + } + if self.max_items != 0 { + len += 1; + } + if !self.items.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("flow.Inference.Array", len)?; + if self.min_items != 0 { + struct_ser.serialize_field("minItems", &self.min_items)?; + } + if self.has_max_items { + struct_ser.serialize_field("hasMaxItems", &self.has_max_items)?; + } + if self.max_items != 0 { + struct_ser.serialize_field("maxItems", &self.max_items)?; + } + if !self.items.is_empty() { + struct_ser.serialize_field("items", &self.items)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for inference::Array { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "min_items", + "minItems", + "has_max_items", + "hasMaxItems", + "max_items", + "maxItems", + "items", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + MinItems, + HasMaxItems, + MaxItems, + Items, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "minItems" | "min_items" => Ok(GeneratedField::MinItems), + "hasMaxItems" | "has_max_items" => Ok(GeneratedField::HasMaxItems), + "maxItems" | "max_items" => Ok(GeneratedField::MaxItems), + "items" => Ok(GeneratedField::Items), + _ => Err(serde::de::Error::unknown_field(value, FIELDS)), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = inference::Array; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct flow.Inference.Array") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut min_items__ = None; + let mut has_max_items__ = None; + let mut max_items__ = None; + let mut items__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::MinItems => { + if min_items__.is_some() { + return Err(serde::de::Error::duplicate_field("minItems")); + } + min_items__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::HasMaxItems => { + if has_max_items__.is_some() { + return Err(serde::de::Error::duplicate_field("hasMaxItems")); + } + has_max_items__ = Some(map_.next_value()?); + } + GeneratedField::MaxItems => { + if max_items__.is_some() { + return Err(serde::de::Error::duplicate_field("maxItems")); + } + max_items__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Items => { + if items__.is_some() { + return Err(serde::de::Error::duplicate_field("items")); + } + items__ = Some(map_.next_value()?); + } + } + } + Ok(inference::Array { + min_items: min_items__.unwrap_or_default(), + has_max_items: has_max_items__.unwrap_or_default(), + max_items: max_items__.unwrap_or_default(), + items: items__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("flow.Inference.Array", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for inference::Exists { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/crates/proto-flow/tests/regression.rs b/crates/proto-flow/tests/regression.rs index 573740ad1e..4c3abb0dca 100644 --- a/crates/proto-flow/tests/regression.rs +++ b/crates/proto-flow/tests/regression.rs @@ -31,6 +31,12 @@ fn ex_projections() -> Vec { has_maximum: false, maximum: 0.0, }), + array: Some(inference::Array { + min_items: 10, + has_max_items: true, + max_items: 20, + items: vec!["null".to_string(), "integer".to_string()], + }), }), }] } diff --git a/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap b/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap index 706064b5ce..ab87fa0dcf 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_request_json.snap @@ -54,6 +54,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } @@ -141,6 +150,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } @@ -276,6 +294,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } diff --git a/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap b/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap index edbba65993..63d374e286 100644 --- a/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__capture_request_proto.snap @@ -5,132 +5,136 @@ expression: proto_test(msg) |0a150807 12117b22 73706563 223a2263| ......{"spec":"c 00000000 |6f6e6669 67227d12 19080712 157b2264| onfig"}......{"d 00000010 |6973636f 76657222 3a22636f 6e666967| iscover":"config 00000020 -|227d1ae2 030a1076 616c6964 6174652f| "}.....validate/ 00000030 +|227d1afa 030a1076 616c6964 6174652f| "}.....validate/ 00000030 |63617074 75726510 071a157b 2276616c| capture....{"val 00000040 |69646174 65223a22 636f6e66 6967227d| idate":"config"} 00000050 -|22a7030a 157b2272 65736f75 72636522| "....{"resource" 00000060 -|3a22636f 6e666967 227d128b 030a1161| :"config"}.....a 00000070 +|22bf030a 157b2272 65736f75 72636522| "....{"resource" 00000060 +|3a22636f 6e666967 227d12a3 030a1161| :"config"}.....a 00000070 |636d6543 6f2f636f 6c6c6563 74696f6e| cmeCo/collection 00000080 |1a082f6b 65792f6f 6e651a08 2f6b6579| ../key/one../key 00000090 |2f74776f 220b2f5f 6d657461 2f757569| /two"./_meta/uui 000000a0 |642a0474 7970652a 06726567 696f6e32| d*.type*.region2 000000b0 -|6a0a092f 6a736f6e 2f707472 1207612d| j../json/ptr..a- 000000c0 -|6669656c 64200132 520a0769 6e746567| field .2R..integ 000000d0 -|65720a06 73747269 6e671a13 1a037479| er..string....ty 000000e0 -|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 000000f0 -|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000100 -|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 00000110 -|08011100 00000000 408fc03a 0e7b2261| ........@..:.{"a 00000120 -|636b223a 22747275 65227d42 127b2277| ck":"true"}B.{"w 00000130 -|72697465 223a2273 6368656d 61227d4a| rite":"schema"}J 00000140 -|a3010a12 70617274 6974696f 6e2f7465| ....partition/te 00000150 -|6d706c61 74651003 1a400a1e 0a0f6573| mplate...@....es 00000160 -|74756172 792e6465 762f666f 6f120b6c| tuary.dev/foo..l 00000170 -|6162656c 2d76616c 75650a1e 0a0f6573| abel-value....es 00000180 -|74756172 792e6465 762f6261 72120b6f| tuary.dev/bar..o 00000190 -|74686572 2d76616c 75652242 08e9ec06| ther-value"B.... 000001a0 -|10031a12 73333a2f 2f627563 6b65742f| ....s3://bucket/ 000001b0 -|70726566 69782203 08ac0232 07083e10| prefix"....2..>. 000001c0 -|80e59a77 3a185061 74687b7b 506f7374| ...w:.Path{{Post 000001d0 -|6669782e 54656d70 6c617465 7d7d3004| fix.Template}}0. 000001e0 -|38cfb0f5 015a117b 22726561 64223a22| 8....Z.{"read":" 000001f0 -|73636865 6d61227d 1801320b 31313a32| schema"}..2.11:2 00000200 -|323a3333 3a343422 eb050ace 050a0e61| 2:33:44".......a 00000210 -|636d6543 6f2f6361 70747572 6510071a| cmeCo/capture... 00000220 -|197b2263 61707475 7265223a 7b22636f| .{"capture":{"co 00000230 -|6e666967 223a3432 7d7d22cb 030a157b| nfig":42}}"....{ 00000240 -|22726573 6f757263 65223a22 636f6e66| "resource":"conf 00000250 -|6967227d 1204736f 6d651204 70617468| ig"}..some..path 00000260 -|1a8b030a 1161636d 65436f2f 636f6c6c| .....acmeCo/coll 00000270 -|65637469 6f6e1a08 2f6b6579 2f6f6e65| ection../key/one 00000280 -|1a082f6b 65792f74 776f220b 2f5f6d65| ../key/two"./_me 00000290 -|74612f75 7569642a 04747970 652a0672| ta/uuid*.type*.r 000002a0 -|6567696f 6e326a0a 092f6a73 6f6e2f70| egion2j../json/p 000002b0 -|74721207 612d6669 656c6420 0132520a| tr..a-field .2R. 000002c0 -|07696e74 65676572 0a067374 72696e67| .integer..string 000002d0 -|1a131a03 74797022 04646174 6530b960| ....typ".date0.` 000002e0 -|3a03656e 63220574 69746c65 2a046465| :.enc".title*.de 000002f0 -|7363320e 7b226465 66223a22 61756c74| sc2.{"def":"ault 00000300 -|227d4001 4a0b0801 11000000 0000408f| "}@.J.........@. 00000310 -|c03a0e7b 2261636b 223a2274 72756522| .:.{"ack":"true" 00000320 -|7d42127b 22777269 7465223a 22736368| }B.{"write":"sch 00000330 -|656d6122 7d4aa301 0a127061 72746974| ema"}J....partit 00000340 -|696f6e2f 74656d70 6c617465 10031a40| ion/template...@ 00000350 -|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000360 -|666f6f12 0b6c6162 656c2d76 616c7565| foo..label-value 00000370 -|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000380 -|62617212 0b6f7468 65722d76 616c7565| bar..other-value 00000390 -|224208e9 ec061003 1a127333 3a2f2f62| "B........s3://b 000003a0 -|75636b65 742f7072 65666978 220308ac| ucket/prefix"... 000003b0 -|02320708 3e1080e5 9a773a18 50617468| .2..>....w:.Path 000003c0 -|7b7b506f 73746669 782e5465 6d706c61| {{Postfix.Templa 000003d0 -|74657d7d 300438cf b0f5015a 117b2272| te}}0.8....Z.{"r 000003e0 -|65616422 3a227363 68656d61 227d2003| ead":"schema"} . 000003f0 -|2a166125 32466364 63253246 7461626c| *.a%2Fcdc%2Ftabl 00000400 -|652b6261 7a2e7633 28ac0232 80010a0e| e+baz.v3(..2.... 00000410 -|73686172 642f7465 6d706c61 74651a0f| shard/template.. 00000420 -|7265636f 76657279 2f707265 66697822| recovery/prefix" 00000430 -|0b68696e 742f7072 65666978 28033202| .hint/prefix(.2. 00000440 -|083c4801 52400a1e 0a0f6573 74756172| .....w:.Path{{ 000006b0 -|506f7374 6669782e 54656d70 6c617465| Postfix.Template 000006c0 -|7d7d3004 38cfb0f5 015a117b 22726561| }}0.8....Z.{"rea 000006d0 -|64223a22 73636865 6d61227d 20032a16| d":"schema"} .*. 000006e0 -|61253246 63646325 32467461 626c652b| a%2Fcdc%2Ftable+ 000006f0 -|62617a2e 763328ac 02328001 0a0e7368| baz.v3(..2....sh 00000700 -|6172642f 74656d70 6c617465 1a0f7265| ard/template..re 00000710 -|636f7665 72792f70 72656669 78220b68| covery/prefix".h 00000720 -|696e742f 70726566 69782803 3202083c| int/prefix(.2..< 00000730 -|48015240 0a1e0a0f 65737475 6172792e| H.R@....estuary. 00000740 -|6465762f 666f6f12 0b6c6162 656c2d76| dev/foo..label-v 00000750 -|616c7565 0a1e0a0f 65737475 6172792e| alue....estuary. 00000760 -|6465762f 62617212 0b6f7468 65722d76| dev/bar..other-v 00000770 -|616c7565 608cdc02 68e9ec06 3a380a11| alue`...h...:8.. 00000780 -|7265636f 76657279 2f74656d 706c6174| recovery/templat 00000790 -|65100322 21088080 4010041a 1473333a| e.."!...@....s3: 000007a0 -|2f2f6275 636b6574 2f726563 6f766572| //bucket/recover 000007b0 -|79220308 ac02420c 08903f12 05687474| y"....B...?..htt 000007c0 -|70731801 420308a8 46120b31 313a3232| ps..B...F..11:22 000007d0 -|3a33333a 34341a14 15332211 001d7766| :33:44...3"...wf 000007e0 -|554425bb aa99882d ffeeddcc 221a7b22| UD%....-....".{" 000007f0 -|636f6e6e 6563746f 72223a7b 22737461| connector":{"sta 00000800 -|7465223a 34327d7d 32020820 a2060612| te":42}}2.. .... 00000810 -|02486918 01| .Hi.. 00000820 - 00000825 +|81010a09 2f6a736f 6e2f7074 72120761| ..../json/ptr..a 000000c0 +|2d666965 6c642001 32690a07 696e7465| -field .2i..inte 000000d0 +|6765720a 06737472 696e671a 131a0374| ger..string....t 000000e0 +|79702204 64617465 30b9603a 03656e63| yp".date0.`:.enc 000000f0 +|22057469 746c652a 04646573 63320e7b| ".title*.desc2.{ 00000100 +|22646566 223a2261 756c7422 7d40014a| "def":"ault"}@.J 00000110 +|0b080111 00000000 00408fc0 5215080a| .........@..R... 00000120 +|10011814 22046e75 6c6c2207 696e7465| ....".null".inte 00000130 +|6765723a 0e7b2261 636b223a 22747275| ger:.{"ack":"tru 00000140 +|65227d42 127b2277 72697465 223a2273| e"}B.{"write":"s 00000150 +|6368656d 61227d4a a3010a12 70617274| chema"}J....part 00000160 +|6974696f 6e2f7465 6d706c61 74651003| ition/template.. 00000170 +|1a400a1e 0a0f6573 74756172 792e6465| .@....estuary.de 00000180 +|762f666f 6f120b6c 6162656c 2d76616c| v/foo..label-val 00000190 +|75650a1e 0a0f6573 74756172 792e6465| ue....estuary.de 000001a0 +|762f6261 72120b6f 74686572 2d76616c| v/bar..other-val 000001b0 +|75652242 08e9ec06 10031a12 73333a2f| ue"B........s3:/ 000001c0 +|2f627563 6b65742f 70726566 69782203| /bucket/prefix". 000001d0 +|08ac0232 07083e10 80e59a77 3a185061| ...2..>....w:.Pa 000001e0 +|74687b7b 506f7374 6669782e 54656d70| th{{Postfix.Temp 000001f0 +|6c617465 7d7d3004 38cfb0f5 015a117b| late}}0.8....Z.{ 00000200 +|22726561 64223a22 73636865 6d61227d| "read":"schema"} 00000210 +|1801320b 31313a32 323a3333 3a343422| ..2.11:22:33:44" 00000220 +|83060ae6 050a0e61 636d6543 6f2f6361| .......acmeCo/ca 00000230 +|70747572 6510071a 197b2263 61707475| pture....{"captu 00000240 +|7265223a 7b22636f 6e666967 223a3432| re":{"config":42 00000250 +|7d7d22e3 030a157b 22726573 6f757263| }}"....{"resourc 00000260 +|65223a22 636f6e66 6967227d 1204736f| e":"config"}..so 00000270 +|6d651204 70617468 1aa3030a 1161636d| me..path.....acm 00000280 +|65436f2f 636f6c6c 65637469 6f6e1a08| eCo/collection.. 00000290 +|2f6b6579 2f6f6e65 1a082f6b 65792f74| /key/one../key/t 000002a0 +|776f220b 2f5f6d65 74612f75 7569642a| wo"./_meta/uuid* 000002b0 +|04747970 652a0672 6567696f 6e328101| .type*.region2.. 000002c0 +|0a092f6a 736f6e2f 70747212 07612d66| ../json/ptr..a-f 000002d0 +|69656c64 20013269 0a07696e 74656765| ield .2i..intege 000002e0 +|720a0673 7472696e 671a131a 03747970| r..string....typ 000002f0 +|22046461 746530b9 603a0365 6e632205| ".date0.`:.enc". 00000300 +|7469746c 652a0464 65736332 0e7b2264| title*.desc2.{"d 00000310 +|6566223a 2261756c 74227d40 014a0b08| ef":"ault"}@.J.. 00000320 +|01110000 00000040 8fc05215 080a1001| .......@..R..... 00000330 +|18142204 6e756c6c 2207696e 74656765| ..".null".intege 00000340 +|723a0e7b 2261636b 223a2274 72756522| r:.{"ack":"true" 00000350 +|7d42127b 22777269 7465223a 22736368| }B.{"write":"sch 00000360 +|656d6122 7d4aa301 0a127061 72746974| ema"}J....partit 00000370 +|696f6e2f 74656d70 6c617465 10031a40| ion/template...@ 00000380 +|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000390 +|666f6f12 0b6c6162 656c2d76 616c7565| foo..label-value 000003a0 +|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 000003b0 +|62617212 0b6f7468 65722d76 616c7565| bar..other-value 000003c0 +|224208e9 ec061003 1a127333 3a2f2f62| "B........s3://b 000003d0 +|75636b65 742f7072 65666978 220308ac| ucket/prefix"... 000003e0 +|02320708 3e1080e5 9a773a18 50617468| .2..>....w:.Path 000003f0 +|7b7b506f 73746669 782e5465 6d706c61| {{Postfix.Templa 00000400 +|74657d7d 300438cf b0f5015a 117b2272| te}}0.8....Z.{"r 00000410 +|65616422 3a227363 68656d61 227d2003| ead":"schema"} . 00000420 +|2a166125 32466364 63253246 7461626c| *.a%2Fcdc%2Ftabl 00000430 +|652b6261 7a2e7633 28ac0232 80010a0e| e+baz.v3(..2.... 00000440 +|73686172 642f7465 6d706c61 74651a0f| shard/template.. 00000450 +|7265636f 76657279 2f707265 66697822| recovery/prefix" 00000460 +|0b68696e 742f7072 65666978 28033202| .hint/prefix(.2. 00000470 +|083c4801 52400a1e 0a0f6573 74756172| .....w 000006f0 +|3a185061 74687b7b 506f7374 6669782e| :.Path{{Postfix. 00000700 +|54656d70 6c617465 7d7d3004 38cfb0f5| Template}}0.8... 00000710 +|015a117b 22726561 64223a22 73636865| .Z.{"read":"sche 00000720 +|6d61227d 20032a16 61253246 63646325| ma"} .*.a%2Fcdc% 00000730 +|32467461 626c652b 62617a2e 763328ac| 2Ftable+baz.v3(. 00000740 +|02328001 0a0e7368 6172642f 74656d70| .2....shard/temp 00000750 +|6c617465 1a0f7265 636f7665 72792f70| late..recovery/p 00000760 +|72656669 78220b68 696e742f 70726566| refix".hint/pref 00000770 +|69782803 3202083c 48015240 0a1e0a0f| ix(.2......w:.P 000001a0 -|6174687b 7b506f73 74666978 2e54656d| ath{{Postfix.Tem 000001b0 -|706c6174 657d7d30 0438cfb0 f5015a11| plate}}0.8....Z. 000001c0 -|7b227265 6164223a 22736368 656d6122| {"read":"schema" 000001d0 -|7d20032a 16612532 46636463 25324674| } .*.a%2Fcdc%2Ft 000001e0 -|61626c65 2b62617a 2e763328 ac023280| able+baz.v3(..2. 000001f0 -|010a0e73 68617264 2f74656d 706c6174| ...shard/templat 00000200 -|651a0f72 65636f76 6572792f 70726566| e..recovery/pref 00000210 -|6978220b 68696e74 2f707265 66697828| ix".hint/prefix( 00000220 -|03320208 3c480152 400a1e0a 0f657374| .2.. 000001b0 +|1080e59a 773a1850 6174687b 7b506f73| ....w:.Path{{Pos 000001c0 +|74666978 2e54656d 706c6174 657d7d30| tfix.Template}}0 000001d0 +|0438cfb0 f5015a11 7b227265 6164223a| .8....Z.{"read": 000001e0 +|22736368 656d6122 7d20032a 16612532| "schema"} .*.a%2 000001f0 +|46636463 25324674 61626c65 2b62617a| Fcdc%2Ftable+baz 00000200 +|2e763328 ac023280 010a0e73 68617264| .v3(..2....shard 00000210 +|2f74656d 706c6174 651a0f72 65636f76| /template..recov 00000220 +|6572792f 70726566 6978220b 68696e74| ery/prefix".hint 00000230 +|2f707265 66697828 03320208 3c480152| /prefix(.2......w:.Path{{P 00000150 -|6f737466 69782e54 656d706c 6174657d| ostfix.Template} 00000160 -|7d300438 cfb0f501 5a117b22 72656164| }0.8....Z.{"read 00000170 -|223a2273 6368656d 61227d| ":"schema"} 00000180 - 0000018b +|6f6e3281 010a092f 6a736f6e 2f707472| on2..../json/ptr 00000040 +|1207612d 6669656c 64200132 690a0769| ..a-field .2i..i 00000050 +|6e746567 65720a06 73747269 6e671a13| nteger..string.. 00000060 +|1a037479 70220464 61746530 b9603a03| ..typ".date0.`:. 00000070 +|656e6322 05746974 6c652a04 64657363| enc".title*.desc 00000080 +|320e7b22 64656622 3a226175 6c74227d| 2.{"def":"ault"} 00000090 +|40014a0b 08011100 00000000 408fc052| @.J.........@..R 000000a0 +|15080a10 01181422 046e756c 6c220769| .......".null".i 000000b0 +|6e746567 65723a0e 7b226163 6b223a22| nteger:.{"ack":" 000000c0 +|74727565 227d4212 7b227772 69746522| true"}B.{"write" 000000d0 +|3a227363 68656d61 227d4aa3 010a1270| :"schema"}J....p 000000e0 +|61727469 74696f6e 2f74656d 706c6174| artition/templat 000000f0 +|6510031a 400a1e0a 0f657374 75617279| e...@....estuary 00000100 +|2e646576 2f666f6f 120b6c61 62656c2d| .dev/foo..label- 00000110 +|76616c75 650a1e0a 0f657374 75617279| value....estuary 00000120 +|2e646576 2f626172 120b6f74 6865722d| .dev/bar..other- 00000130 +|76616c75 65224208 e9ec0610 031a1273| value"B........s 00000140 +|333a2f2f 6275636b 65742f70 72656669| 3://bucket/prefi 00000150 +|78220308 ac023207 083e1080 e59a773a| x"....2..>....w: 00000160 +|18506174 687b7b50 6f737466 69782e54| .Path{{Postfix.T 00000170 +|656d706c 6174657d 7d300438 cfb0f501| emplate}}0.8.... 00000180 +|5a117b22 72656164 223a2273 6368656d| Z.{"read":"schem 00000190 +|61227d| a"} 000001a0 + 000001a3 diff --git a/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap b/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap index f9f1439eef..11cf1f41d2 100644 --- a/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__derivation_spec_json.snap @@ -38,6 +38,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } @@ -113,6 +122,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } diff --git a/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap b/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap index 2a3d19c440..a656149bc2 100644 --- a/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__derivation_spec_proto.snap @@ -6,80 +6,83 @@ expression: proto_test(msg) |696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 00000010 |6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 00000020 |75756964 2a047479 70652a06 72656769| uuid*.type*.regi 00000030 -|6f6e326a 0a092f6a 736f6e2f 70747212| on2j../json/ptr. 00000040 -|07612d66 69656c64 20013252 0a07696e| .a-field .2R..in 00000050 -|74656765 720a0673 7472696e 671a131a| teger..string... 00000060 -|03747970 22046461 746530b9 603a0365| .typ".date0.`:.e 00000070 -|6e632205 7469746c 652a0464 65736332| nc".title*.desc2 00000080 -|0e7b2264 6566223a 2261756c 74227d40| .{"def":"ault"}@ 00000090 -|014a0b08 01110000 00000040 8fc03a0e| .J.........@..:. 000000a0 -|7b226163 6b223a22 74727565 227d4212| {"ack":"true"}B. 000000b0 -|7b227772 69746522 3a227363 68656d61| {"write":"schema 000000c0 -|227d4aa3 010a1270 61727469 74696f6e| "}J....partition 000000d0 -|2f74656d 706c6174 6510031a 400a1e0a| /template...@... 000000e0 -|0f657374 75617279 2e646576 2f666f6f| .estuary.dev/foo 000000f0 -|120b6c61 62656c2d 76616c75 650a1e0a| ..label-value... 00000100 -|0f657374 75617279 2e646576 2f626172| .estuary.dev/bar 00000110 -|120b6f74 6865722d 76616c75 65224208| ..other-value"B. 00000120 -|e9ec0610 031a1273 333a2f2f 6275636b| .......s3://buck 00000130 -|65742f70 72656669 78220308 ac023207| et/prefix"....2. 00000140 -|083e1080 e59a773a 18506174 687b7b50| .>....w:.Path{{P 00000150 -|6f737466 69782e54 656d706c 6174657d| ostfix.Template} 00000160 -|7d300438 cfb0f501 5a117b22 72656164| }0.8....Z.{"read 00000170 -|223a2273 6368656d 61227d62 f0060801| ":"schema"}b.... 00000180 -|121c7b22 64657269 76617469 6f6e223a| ..{"derivation": 00000190 -|7b22636f 6e666967 223a3432 7d7d1af9| {"config":42}}.. 000001a0 -|040a0e74 72616e73 666f726d 5f6e616d| ...transform_nam 000001b0 -|65128b03 0a116163 6d65436f 2f636f6c| e.....acmeCo/col 000001c0 -|6c656374 696f6e1a 082f6b65 792f6f6e| lection../key/on 000001d0 -|651a082f 6b65792f 74776f22 0b2f5f6d| e../key/two"./_m 000001e0 -|6574612f 75756964 2a047479 70652a06| eta/uuid*.type*. 000001f0 -|72656769 6f6e326a 0a092f6a 736f6e2f| region2j../json/ 00000200 -|70747212 07612d66 69656c64 20013252| ptr..a-field .2R 00000210 -|0a07696e 74656765 720a0673 7472696e| ..integer..strin 00000220 -|671a131a 03747970 22046461 746530b9| g....typ".date0. 00000230 -|603a0365 6e632205 7469746c 652a0464| `:.enc".title*.d 00000240 -|65736332 0e7b2264 6566223a 2261756c| esc2.{"def":"aul 00000250 -|74227d40 014a0b08 01110000 00000040| t"}@.J.........@ 00000260 -|8fc03a0e 7b226163 6b223a22 74727565| ..:.{"ack":"true 00000270 -|227d4212 7b227772 69746522 3a227363| "}B.{"write":"sc 00000280 -|68656d61 227d4aa3 010a1270 61727469| hema"}J....parti 00000290 -|74696f6e 2f74656d 706c6174 6510031a| tion/template... 000002a0 -|400a1e0a 0f657374 75617279 2e646576| @....estuary.dev 000002b0 -|2f666f6f 120b6c61 62656c2d 76616c75| /foo..label-valu 000002c0 -|650a1e0a 0f657374 75617279 2e646576| e....estuary.dev 000002d0 -|2f626172 120b6f74 6865722d 76616c75| /bar..other-valu 000002e0 -|65224208 e9ec0610 031a1273 333a2f2f| e"B........s3:// 000002f0 -|6275636b 65742f70 72656669 78220308| bucket/prefix".. 00000300 -|ac023207 083e1080 e59a773a 18506174| ..2..>....w:.Pat 00000310 -|687b7b50 6f737466 69782e54 656d706c| h{{Postfix.Templ 00000320 -|6174657d 7d300438 cfb0f501 5a117b22| ate}}0.8....Z.{" 00000330 -|72656164 223a2273 6368656d 61227d1a| read":"schema"}. 00000340 -|5b0a400a 1e0a0f65 73747561 72792e64| [.@....estuary.d 00000350 -|65762f66 6f6f120b 6c616265 6c2d7661| ev/foo..label-va 00000360 -|6c75650a 1e0a0f65 73747561 72792e64| lue....estuary.d 00000370 -|65762f62 6172120b 6f746865 722d7661| ev/bar..other-va 00000380 -|6c756512 170a150a 086d792d 6c616265| lue......my-labe 00000390 -|6c120770 72656669 782f1801 2001280e| l..prefix/.. .(. 000003a0 -|32082f73 68756666 6c653204 2f6b6579| 2./shuffle2./key 000003b0 -|3a182253 454c4543 54202473 68756666| :."SELECT $shuff 000003c0 -|6c652c20 246b6579 3b224213 7b226c61| le, $key;"B.{"la 000003d0 -|6d626461 223a2263 6f6e6669 67227d48| mbda":"config"}H 000003e0 -|01522564 65726976 652f612f 636f6c6c| .R%derive/a/coll 000003f0 -|65637469 6f6e2f74 72616e73 666f726d| ection/transform 00000400 -|5f6e616d 652e7632 5a0608cb c8d6a606| _name.v2Z....... 00000410 -|62060880 888ba106 68022202 03022a80| b.......h."...*. 00000420 -|010a0e73 68617264 2f74656d 706c6174| ...shard/templat 00000430 -|651a0f72 65636f76 6572792f 70726566| e..recovery/pref 00000440 -|6978220b 68696e74 2f707265 66697828| ix".hint/prefix( 00000450 -|03320208 3c480152 400a1e0a 0f657374| .2......w: 00000160 +|18506174 687b7b50 6f737466 69782e54| .Path{{Postfix.T 00000170 +|656d706c 6174657d 7d300438 cfb0f501| emplate}}0.8.... 00000180 +|5a117b22 72656164 223a2273 6368656d| Z.{"read":"schem 00000190 +|61227d62 88070801 121c7b22 64657269| a"}b......{"deri 000001a0 +|76617469 6f6e223a 7b22636f 6e666967| vation":{"config 000001b0 +|223a3432 7d7d1a91 050a0e74 72616e73| ":42}}.....trans 000001c0 +|666f726d 5f6e616d 6512a303 0a116163| form_name.....ac 000001d0 +|6d65436f 2f636f6c 6c656374 696f6e1a| meCo/collection. 000001e0 +|082f6b65 792f6f6e 651a082f 6b65792f| ./key/one../key/ 000001f0 +|74776f22 0b2f5f6d 6574612f 75756964| two"./_meta/uuid 00000200 +|2a047479 70652a06 72656769 6f6e3281| *.type*.region2. 00000210 +|010a092f 6a736f6e 2f707472 1207612d| .../json/ptr..a- 00000220 +|6669656c 64200132 690a0769 6e746567| field .2i..integ 00000230 +|65720a06 73747269 6e671a13 1a037479| er..string....ty 00000240 +|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 00000250 +|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000260 +|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 00000270 +|08011100 00000000 408fc052 15080a10| ........@..R.... 00000280 +|01181422 046e756c 6c220769 6e746567| ...".null".integ 00000290 +|65723a0e 7b226163 6b223a22 74727565| er:.{"ack":"true 000002a0 +|227d4212 7b227772 69746522 3a227363| "}B.{"write":"sc 000002b0 +|68656d61 227d4aa3 010a1270 61727469| hema"}J....parti 000002c0 +|74696f6e 2f74656d 706c6174 6510031a| tion/template... 000002d0 +|400a1e0a 0f657374 75617279 2e646576| @....estuary.dev 000002e0 +|2f666f6f 120b6c61 62656c2d 76616c75| /foo..label-valu 000002f0 +|650a1e0a 0f657374 75617279 2e646576| e....estuary.dev 00000300 +|2f626172 120b6f74 6865722d 76616c75| /bar..other-valu 00000310 +|65224208 e9ec0610 031a1273 333a2f2f| e"B........s3:// 00000320 +|6275636b 65742f70 72656669 78220308| bucket/prefix".. 00000330 +|ac023207 083e1080 e59a773a 18506174| ..2..>....w:.Pat 00000340 +|687b7b50 6f737466 69782e54 656d706c| h{{Postfix.Templ 00000350 +|6174657d 7d300438 cfb0f501 5a117b22| ate}}0.8....Z.{" 00000360 +|72656164 223a2273 6368656d 61227d1a| read":"schema"}. 00000370 +|5b0a400a 1e0a0f65 73747561 72792e64| [.@....estuary.d 00000380 +|65762f66 6f6f120b 6c616265 6c2d7661| ev/foo..label-va 00000390 +|6c75650a 1e0a0f65 73747561 72792e64| lue....estuary.d 000003a0 +|65762f62 6172120b 6f746865 722d7661| ev/bar..other-va 000003b0 +|6c756512 170a150a 086d792d 6c616265| lue......my-labe 000003c0 +|6c120770 72656669 782f1801 2001280e| l..prefix/.. .(. 000003d0 +|32082f73 68756666 6c653204 2f6b6579| 2./shuffle2./key 000003e0 +|3a182253 454c4543 54202473 68756666| :."SELECT $shuff 000003f0 +|6c652c20 246b6579 3b224213 7b226c61| le, $key;"B.{"la 00000400 +|6d626461 223a2263 6f6e6669 67227d48| mbda":"config"}H 00000410 +|01522564 65726976 652f612f 636f6c6c| .R%derive/a/coll 00000420 +|65637469 6f6e2f74 72616e73 666f726d| ection/transform 00000430 +|5f6e616d 652e7632 5a0608cb c8d6a606| _name.v2Z....... 00000440 +|62060880 888ba106 68022202 03022a80| b.......h."...*. 00000450 +|010a0e73 68617264 2f74656d 706c6174| ...shard/templat 00000460 +|651a0f72 65636f76 6572792f 70726566| e..recovery/pref 00000470 +|6978220b 68696e74 2f707265 66697828| ix".hint/prefix( 00000480 +|03320208 3c480152 400a1e0a 0f657374| .2......w:.P 00000180 -|6174687b 7b506f73 74666978 2e54656d| ath{{Postfix.Tem 00000190 -|706c6174 657d7d30 0438cfb0 f5015a11| plate}}0.8....Z. 000001a0 -|7b227265 6164223a 22736368 656d6122| {"read":"schema" 000001b0 -|7d22c803 0a0b7374 61626c65 5f6e616d| }"....stable_nam 000001c0 -|65128b03 0a116163 6d65436f 2f636f6c| e.....acmeCo/col 000001d0 -|6c656374 696f6e1a 082f6b65 792f6f6e| lection../key/on 000001e0 -|651a082f 6b65792f 74776f22 0b2f5f6d| e../key/two"./_m 000001f0 -|6574612f 75756964 2a047479 70652a06| eta/uuid*.type*. 00000200 -|72656769 6f6e326a 0a092f6a 736f6e2f| region2j../json/ 00000210 -|70747212 07612d66 69656c64 20013252| ptr..a-field .2R 00000220 -|0a07696e 74656765 720a0673 7472696e| ..integer..strin 00000230 -|671a131a 03747970 22046461 746530b9| g....typ".date0. 00000240 -|603a0365 6e632205 7469746c 652a0464| `:.enc".title*.d 00000250 -|65736332 0e7b2264 6566223a 2261756c| esc2.{"def":"aul 00000260 -|74227d40 014a0b08 01110000 00000040| t"}@.J.........@ 00000270 -|8fc03a0e 7b226163 6b223a22 74727565| ..:.{"ack":"true 00000280 -|227d4212 7b227772 69746522 3a227363| "}B.{"write":"sc 00000290 -|68656d61 227d4aa3 010a1270 61727469| hema"}J....parti 000002a0 -|74696f6e 2f74656d 706c6174 6510031a| tion/template... 000002b0 -|400a1e0a 0f657374 75617279 2e646576| @....estuary.dev 000002c0 -|2f666f6f 120b6c61 62656c2d 76616c75| /foo..label-valu 000002d0 -|650a1e0a 0f657374 75617279 2e646576| e....estuary.dev 000002e0 -|2f626172 120b6f74 6865722d 76616c75| /bar..other-valu 000002f0 -|65224208 e9ec0610 031a1273 333a2f2f| e"B........s3:// 00000300 -|6275636b 65742f70 72656669 78220308| bucket/prefix".. 00000310 -|ac023207 083e1080 e59a773a 18506174| ..2..>....w:.Pat 00000320 -|687b7b50 6f737466 69782e54 656d706c| h{{Postfix.Templ 00000330 -|6174657d 7d300438 cfb0f501 5a117b22| ate}}0.8....Z.{" 00000340 -|72656164 223a2273 6368656d 61227d1a| read":"schema"}. 00000350 -|147b2273 68756666 6c65223a 22636f6e| .{"shuffle":"con 00000360 -|66696722 7d22137b 226c616d 62646122| fig"}".{"lambda" 00000370 -|3a22636f 6e666967 227d2802 2a020102| :"config"}(.*... 00000380 -|32146669 6c653a2f 2f2f7072 6f6a6563| 2.file:///projec 00000390 -|742f726f 6f743a32 0a182f75 73696e67| t/root:2../using 000003a0 -|2f747970 65736372 6970742f 6d6f6475| /typescript/modu 000003b0 -|6c651216 66696c65 3a2f2f2f 70617468| le..file:///path 000003c0 -|2f746f2f 696d706f 72744a0b 30303a31| /to/importJ.00:1 000003d0 -|313a3232 3a33331a cd030a8b 030a1161| 1:22:33........a 000003e0 -|636d6543 6f2f636f 6c6c6563 74696f6e| cmeCo/collection 000003f0 -|1a082f6b 65792f6f 6e651a08 2f6b6579| ../key/one../key 00000400 -|2f74776f 220b2f5f 6d657461 2f757569| /two"./_meta/uui 00000410 -|642a0474 7970652a 06726567 696f6e32| d*.type*.region2 00000420 -|6a0a092f 6a736f6e 2f707472 1207612d| j../json/ptr..a- 00000430 -|6669656c 64200132 520a0769 6e746567| field .2R..integ 00000440 -|65720a06 73747269 6e671a13 1a037479| er..string....ty 00000450 -|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 00000460 -|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000470 -|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 00000480 -|08011100 00000000 408fc03a 0e7b2261| ........@..:.{"a 00000490 -|636b223a 22747275 65227d42 127b2277| ck":"true"}B.{"w 000004a0 -|72697465 223a2273 6368656d 61227d4a| rite":"schema"}J 000004b0 -|a3010a12 70617274 6974696f 6e2f7465| ....partition/te 000004c0 -|6d706c61 74651003 1a400a1e 0a0f6573| mplate...@....es 000004d0 -|74756172 792e6465 762f666f 6f120b6c| tuary.dev/foo..l 000004e0 -|6162656c 2d76616c 75650a1e 0a0f6573| abel-value....es 000004f0 -|74756172 792e6465 762f6261 72120b6f| tuary.dev/bar..o 00000500 -|74686572 2d76616c 75652242 08e9ec06| ther-value"B.... 00000510 -|10031a12 73333a2f 2f627563 6b65742f| ....s3://bucket/ 00000520 -|70726566 69782203 08ac0232 07083e10| prefix"....2..>. 00000530 -|80e59a77 3a185061 74687b7b 506f7374| ...w:.Path{{Post 00000540 -|6669782e 54656d70 6c617465 7d7d3004| fix.Template}}0. 00000550 -|38cfb0f5 015a117b 22726561 64223a22| 8....Z.{"read":" 00000560 -|73636865 6d61227d 120b3131 3a32323a| schema"}..11:22: 00000570 -|33333a34 341a1415 33221100 1d776655| 33:44...3"...wfU 00000580 -|4425bbaa 99882dff eeddcc2a 1a7b2263| D%....-....*.{"c 00000590 -|6f6e6e65 63746f72 223a7b22 73746174| onnector":{"stat 000005a0 -|65223a34 327d7d22 3e080212 1209d204| e":42}}">....... 000005b0 -|00000000 0000112e 16000000 0000001a| ................ 000005c0 -|160a095b 74727565 2c33325d 1204564b| ...[true,32]..VK 000005d0 -|1e091885 c39f1522 0e7b2272 65616422| .......".{"read" 000005e0 -|3a22646f 63227d2a 0032660a 640a4a0a| :"doc"}*.2f.d.J. 000005f0 -|15612f72 6561642f 6a6f7572 6e616c3b| .a/read/journal; 00000600 -|73756666 69781231 08b96012 150a0503| suffix.1..`..... 00000610 -|09080507 120c09e3 21000000 00000010| ........!....... 00000620 -|d7081215 0a05070c 662b1d12 0c093501| ........f+....5. 00000630 -|00000000 000010ae 1112160a 0e616e2f| .............an/ 00000640 -|61636b2f 6a6f7572 6e616c12 04030402| ack/journal..... 00000650 -|053a00a2 06061202 48691801| .:......Hi.. 00000660 - 0000066c +|2a067265 67696f6e 3281010a 092f6a73| *.region2..../js 00000070 +|6f6e2f70 74721207 612d6669 656c6420| on/ptr..a-field 00000080 +|0132690a 07696e74 65676572 0a067374| .2i..integer..st 00000090 +|72696e67 1a131a03 74797022 04646174| ring....typ".dat 000000a0 +|6530b960 3a03656e 63220574 69746c65| e0.`:.enc".title 000000b0 +|2a046465 7363320e 7b226465 66223a22| *.desc2.{"def":" 000000c0 +|61756c74 227d4001 4a0b0801 11000000| ault"}@.J....... 000000d0 +|0000408f c0521508 0a100118 1422046e| ..@..R.......".n 000000e0 +|756c6c22 07696e74 65676572 3a0e7b22| ull".integer:.{" 000000f0 +|61636b22 3a227472 7565227d 42127b22| ack":"true"}B.{" 00000100 +|77726974 65223a22 73636865 6d61227d| write":"schema"} 00000110 +|4aa3010a 12706172 74697469 6f6e2f74| J....partition/t 00000120 +|656d706c 61746510 031a400a 1e0a0f65| emplate...@....e 00000130 +|73747561 72792e64 65762f66 6f6f120b| stuary.dev/foo.. 00000140 +|6c616265 6c2d7661 6c75650a 1e0a0f65| label-value....e 00000150 +|73747561 72792e64 65762f62 6172120b| stuary.dev/bar.. 00000160 +|6f746865 722d7661 6c756522 4208e9ec| other-value"B... 00000170 +|0610031a 1273333a 2f2f6275 636b6574| .....s3://bucket 00000180 +|2f707265 66697822 0308ac02 3207083e| /prefix"....2..> 00000190 +|1080e59a 773a1850 6174687b 7b506f73| ....w:.Path{{Pos 000001a0 +|74666978 2e54656d 706c6174 657d7d30| tfix.Template}}0 000001b0 +|0438cfb0 f5015a11 7b227265 6164223a| .8....Z.{"read": 000001c0 +|22736368 656d6122 7d22e003 0a0b7374| "schema"}"....st 000001d0 +|61626c65 5f6e616d 6512a303 0a116163| able_name.....ac 000001e0 +|6d65436f 2f636f6c 6c656374 696f6e1a| meCo/collection. 000001f0 +|082f6b65 792f6f6e 651a082f 6b65792f| ./key/one../key/ 00000200 +|74776f22 0b2f5f6d 6574612f 75756964| two"./_meta/uuid 00000210 +|2a047479 70652a06 72656769 6f6e3281| *.type*.region2. 00000220 +|010a092f 6a736f6e 2f707472 1207612d| .../json/ptr..a- 00000230 +|6669656c 64200132 690a0769 6e746567| field .2i..integ 00000240 +|65720a06 73747269 6e671a13 1a037479| er..string....ty 00000250 +|70220464 61746530 b9603a03 656e6322| p".date0.`:.enc" 00000260 +|05746974 6c652a04 64657363 320e7b22| .title*.desc2.{" 00000270 +|64656622 3a226175 6c74227d 40014a0b| def":"ault"}@.J. 00000280 +|08011100 00000000 408fc052 15080a10| ........@..R.... 00000290 +|01181422 046e756c 6c220769 6e746567| ...".null".integ 000002a0 +|65723a0e 7b226163 6b223a22 74727565| er:.{"ack":"true 000002b0 +|227d4212 7b227772 69746522 3a227363| "}B.{"write":"sc 000002c0 +|68656d61 227d4aa3 010a1270 61727469| hema"}J....parti 000002d0 +|74696f6e 2f74656d 706c6174 6510031a| tion/template... 000002e0 +|400a1e0a 0f657374 75617279 2e646576| @....estuary.dev 000002f0 +|2f666f6f 120b6c61 62656c2d 76616c75| /foo..label-valu 00000300 +|650a1e0a 0f657374 75617279 2e646576| e....estuary.dev 00000310 +|2f626172 120b6f74 6865722d 76616c75| /bar..other-valu 00000320 +|65224208 e9ec0610 031a1273 333a2f2f| e"B........s3:// 00000330 +|6275636b 65742f70 72656669 78220308| bucket/prefix".. 00000340 +|ac023207 083e1080 e59a773a 18506174| ..2..>....w:.Pat 00000350 +|687b7b50 6f737466 69782e54 656d706c| h{{Postfix.Templ 00000360 +|6174657d 7d300438 cfb0f501 5a117b22| ate}}0.8....Z.{" 00000370 +|72656164 223a2273 6368656d 61227d1a| read":"schema"}. 00000380 +|147b2273 68756666 6c65223a 22636f6e| .{"shuffle":"con 00000390 +|66696722 7d22137b 226c616d 62646122| fig"}".{"lambda" 000003a0 +|3a22636f 6e666967 227d2802 2a020102| :"config"}(.*... 000003b0 +|32146669 6c653a2f 2f2f7072 6f6a6563| 2.file:///projec 000003c0 +|742f726f 6f743a32 0a182f75 73696e67| t/root:2../using 000003d0 +|2f747970 65736372 6970742f 6d6f6475| /typescript/modu 000003e0 +|6c651216 66696c65 3a2f2f2f 70617468| le..file:///path 000003f0 +|2f746f2f 696d706f 72744a0b 30303a31| /to/importJ.00:1 00000400 +|313a3232 3a33331a e5030aa3 030a1161| 1:22:33........a 00000410 +|636d6543 6f2f636f 6c6c6563 74696f6e| cmeCo/collection 00000420 +|1a082f6b 65792f6f 6e651a08 2f6b6579| ../key/one../key 00000430 +|2f74776f 220b2f5f 6d657461 2f757569| /two"./_meta/uui 00000440 +|642a0474 7970652a 06726567 696f6e32| d*.type*.region2 00000450 +|81010a09 2f6a736f 6e2f7074 72120761| ..../json/ptr..a 00000460 +|2d666965 6c642001 32690a07 696e7465| -field .2i..inte 00000470 +|6765720a 06737472 696e671a 131a0374| ger..string....t 00000480 +|79702204 64617465 30b9603a 03656e63| yp".date0.`:.enc 00000490 +|22057469 746c652a 04646573 63320e7b| ".title*.desc2.{ 000004a0 +|22646566 223a2261 756c7422 7d40014a| "def":"ault"}@.J 000004b0 +|0b080111 00000000 00408fc0 5215080a| .........@..R... 000004c0 +|10011814 22046e75 6c6c2207 696e7465| ....".null".inte 000004d0 +|6765723a 0e7b2261 636b223a 22747275| ger:.{"ack":"tru 000004e0 +|65227d42 127b2277 72697465 223a2273| e"}B.{"write":"s 000004f0 +|6368656d 61227d4a a3010a12 70617274| chema"}J....part 00000500 +|6974696f 6e2f7465 6d706c61 74651003| ition/template.. 00000510 +|1a400a1e 0a0f6573 74756172 792e6465| .@....estuary.de 00000520 +|762f666f 6f120b6c 6162656c 2d76616c| v/foo..label-val 00000530 +|75650a1e 0a0f6573 74756172 792e6465| ue....estuary.de 00000540 +|762f6261 72120b6f 74686572 2d76616c| v/bar..other-val 00000550 +|75652242 08e9ec06 10031a12 73333a2f| ue"B........s3:/ 00000560 +|2f627563 6b65742f 70726566 69782203| /bucket/prefix". 00000570 +|08ac0232 07083e10 80e59a77 3a185061| ...2..>....w:.Pa 00000580 +|74687b7b 506f7374 6669782e 54656d70| th{{Postfix.Temp 00000590 +|6c617465 7d7d3004 38cfb0f5 015a117b| late}}0.8....Z.{ 000005a0 +|22726561 64223a22 73636865 6d61227d| "read":"schema"} 000005b0 +|120b3131 3a32323a 33333a34 341a1415| ..11:22:33:44... 000005c0 +|33221100 1d776655 4425bbaa 99882dff| 3"...wfUD%....-. 000005d0 +|eeddcc2a 1a7b2263 6f6e6e65 63746f72| ...*.{"connector 000005e0 +|223a7b22 73746174 65223a34 327d7d22| ":{"state":42}}" 000005f0 +|3e080212 1209d204 00000000 0000112e| >............... 00000600 +|16000000 0000001a 160a095b 74727565| ...........[true 00000610 +|2c33325d 1204564b 1e091885 c39f1522| ,32]..VK......." 00000620 +|0e7b2272 65616422 3a22646f 63227d2a| .{"read":"doc"}* 00000630 +|0032660a 640a4a0a 15612f72 6561642f| .2f.d.J..a/read/ 00000640 +|6a6f7572 6e616c3b 73756666 69781231| journal;suffix.1 00000650 +|08b96012 150a0503 09080507 120c09e3| ..`............. 00000660 +|21000000 00000010 d7081215 0a05070c| !............... 00000670 +|662b1d12 0c093501 00000000 000010ae| f+....5......... 00000680 +|1112160a 0e616e2f 61636b2f 6a6f7572| .....an/ack/jour 00000690 +|6e616c12 04030402 053a00a2 06061202| nal......:...... 000006a0 +|48691801| Hi.. 000006b0 + 000006b4 diff --git a/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap b/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap index 8906fb2ead..cc601b5e54 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialization_spec_json.snap @@ -49,6 +49,15 @@ expression: json_test(msg) "numeric": { "hasMinimum": true, "minimum": -1000.0 + }, + "array": { + "minItems": 10, + "hasMaxItems": true, + "maxItems": 20, + "items": [ + "null", + "integer" + ] } } } diff --git a/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap b/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap index cf78d1cb01..605a15e2d3 100644 --- a/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap +++ b/crates/proto-flow/tests/snapshots/regression__materialization_spec_proto.snap @@ -5,62 +5,63 @@ expression: proto_test(msg) |0a166163 6d65436f 2f6d6174 65726961| ..acmeCo/materia 00000000 |6c697a61 74696f6e 10081a1d 7b226d61| lization....{"ma 00000010 |74657269 616c697a 65223a7b 22636f6e| terialize":{"con 00000020 -|66696722 3a34327d 7d22bc05 0a157b22| fig":42}}"....{" 00000030 +|66696722 3a34327d 7d22d405 0a157b22| fig":42}}"....{" 00000030 |7265736f 75726365 223a2263 6f6e6669| resource":"confi 00000040 |67227d12 04736f6d 65120470 6174681a| g"}..some..path. 00000050 -|8b030a11 61636d65 436f2f63 6f6c6c65| ....acmeCo/colle 00000060 +|a3030a11 61636d65 436f2f63 6f6c6c65| ....acmeCo/colle 00000060 |6374696f 6e1a082f 6b65792f 6f6e651a| ction../key/one. 00000070 |082f6b65 792f7477 6f220b2f 5f6d6574| ./key/two"./_met 00000080 |612f7575 69642a04 74797065 2a067265| a/uuid*.type*.re 00000090 -|67696f6e 326a0a09 2f6a736f 6e2f7074| gion2j../json/pt 000000a0 -|72120761 2d666965 6c642001 32520a07| r..a-field .2R.. 000000b0 -|696e7465 6765720a 06737472 696e671a| integer..string. 000000c0 -|131a0374 79702204 64617465 30b9603a| ...typ".date0.`: 000000d0 -|03656e63 22057469 746c652a 04646573| .enc".title*.des 000000e0 -|63320e7b 22646566 223a2261 756c7422| c2.{"def":"ault" 000000f0 -|7d40014a 0b080111 00000000 00408fc0| }@.J.........@.. 00000100 -|3a0e7b22 61636b22 3a227472 7565227d| :.{"ack":"true"} 00000110 -|42127b22 77726974 65223a22 73636865| B.{"write":"sche 00000120 -|6d61227d 4aa3010a 12706172 74697469| ma"}J....partiti 00000130 -|6f6e2f74 656d706c 61746510 031a400a| on/template...@. 00000140 -|1e0a0f65 73747561 72792e64 65762f66| ...estuary.dev/f 00000150 -|6f6f120b 6c616265 6c2d7661 6c75650a| oo..label-value. 00000160 -|1e0a0f65 73747561 72792e64 65762f62| ...estuary.dev/b 00000170 -|6172120b 6f746865 722d7661 6c756522| ar..other-value" 00000180 -|4208e9ec 0610031a 1273333a 2f2f6275| B........s3://bu 00000190 -|636b6574 2f707265 66697822 0308ac02| cket/prefix".... 000001a0 -|3207083e 1080e59a 773a1850 6174687b| 2..>....w:.Path{ 000001b0 -|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000001c0 -|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 000001d0 -|6164223a 22736368 656d6122 7d22550a| ad":"schema"}"U. 000001e0 -|076b6579 2f6f6e65 12077661 6c2f7477| .key/one..val/tw 000001f0 -|6f1a0d66 6c6f775f 646f6375 6d656e74| o..flow_document 00000200 -|221d0a07 615f6669 656c6412 127b2266| "...a_field..{"f 00000210 -|69656c64 223a2263 6f6e6669 67227d22| ield":"config"}" 00000220 -|130a0b6f 74686572 2f666965 6c641204| ...other/field.. 00000230 -|34322e35 3a5b0a40 0a1e0a0f 65737475| 42.5:[.@....estu 00000240 -|6172792e 6465762f 666f6f12 0b6c6162| ary.dev/foo..lab 00000250 -|656c2d76 616c7565 0a1e0a0f 65737475| el-value....estu 00000260 -|6172792e 6465762f 62617212 0b6f7468| ary.dev/bar..oth 00000270 -|65722d76 616c7565 12170a15 0a086d79| er-value......my 00000280 -|2d6c6162 656c1207 70726566 69782f18| -label..prefix/. 00000290 -|0142316d 61746572 69616c69 7a652f61| .B1materialize/a 000002a0 -|636d6543 6f2f6d61 74657269 616c697a| cmeCo/materializ 000002b0 -|6174696f 6e2f736f 6d652532 30706174| ation/some%20pat 000002c0 -|682e7631 48035206 08cbc8d6 a6065a06| h.v1H.R.......Z. 000002d0 -|0880888b a1066001 6a0e736f 6d652532| ......`.j.some%2 000002e0 -|30706174 682e7631 2a80010a 0e736861| 0path.v1*....sha 000002f0 -|72642f74 656d706c 6174651a 0f726563| rd/template..rec 00000300 -|6f766572 792f7072 65666978 220b6869| overy/prefix".hi 00000310 -|6e742f70 72656669 78280332 02083c48| nt/prefix(.2...... 000001c0 +|773a1850 6174687b 7b506f73 74666978| w:.Path{{Postfix 000001d0 +|2e54656d 706c6174 657d7d30 0438cfb0| .Template}}0.8.. 000001e0 +|f5015a11 7b227265 6164223a 22736368| ..Z.{"read":"sch 000001f0 +|656d6122 7d22550a 076b6579 2f6f6e65| ema"}"U..key/one 00000200 +|12077661 6c2f7477 6f1a0d66 6c6f775f| ..val/two..flow_ 00000210 +|646f6375 6d656e74 221d0a07 615f6669| document"...a_fi 00000220 +|656c6412 127b2266 69656c64 223a2263| eld..{"field":"c 00000230 +|6f6e6669 67227d22 130a0b6f 74686572| onfig"}"...other 00000240 +|2f666965 6c641204 34322e35 3a5b0a40| /field..42.5:[.@ 00000250 +|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000260 +|666f6f12 0b6c6162 656c2d76 616c7565| foo..label-value 00000270 +|0a1e0a0f 65737475 6172792e 6465762f| ....estuary.dev/ 00000280 +|62617212 0b6f7468 65722d76 616c7565| bar..other-value 00000290 +|12170a15 0a086d79 2d6c6162 656c1207| ......my-label.. 000002a0 +|70726566 69782f18 0142316d 61746572| prefix/..B1mater 000002b0 +|69616c69 7a652f61 636d6543 6f2f6d61| ialize/acmeCo/ma 000002c0 +|74657269 616c697a 6174696f 6e2f736f| terialization/so 000002d0 +|6d652532 30706174 682e7631 48035206| me%20path.v1H.R. 000002e0 +|08cbc8d6 a6065a06 0880888b a1066001| ......Z.......`. 000002f0 +|6a0e736f 6d652532 30706174 682e7631| j.some%20path.v1 00000300 +|2a80010a 0e736861 72642f74 656d706c| *....shard/templ 00000310 +|6174651a 0f726563 6f766572 792f7072| ate..recovery/pr 00000320 +|65666978 220b6869 6e742f70 72656669| efix".hint/prefi 00000330 +|78280332 02083c48 0152400a 1e0a0f65| x(.2...... 000001b0 -|773a1850 6174687b 7b506f73 74666978| w:.Path{{Postfix 000001c0 -|2e54656d 706c6174 657d7d30 0438cfb0| .Template}}0.8.. 000001d0 -|f5015a11 7b227265 6164223a 22736368| ..Z.{"read":"sch 000001e0 -|656d6122 7d1a1d0a 07615f66 69656c64| ema"}....a_field 000001f0 -|12127b22 6669656c 64223a22 636f6e66| ..{"field":"conf 00000200 -|6967227d 1a130a0b 6f746865 722f6669| ig"}....other/fi 00000210 -|656c6412 0434322e 35200332 0b30303a| eld..42.5 .2.00: 00000220 -|31313a32 323a3333 1ae5070a c8070a16| 11:22:33........ 00000230 -|61636d65 436f2f6d 61746572 69616c69| acmeCo/materiali 00000240 -|7a617469 6f6e1008 1a1d7b22 6d617465| zation....{"mate 00000250 -|7269616c 697a6522 3a7b2263 6f6e6669| rialize":{"confi 00000260 -|67223a34 327d7d22 bc050a15 7b227265| g":42}}"....{"re 00000270 -|736f7572 6365223a 22636f6e 66696722| source":"config" 00000280 -|7d120473 6f6d6512 04706174 681a8b03| }..some..path... 00000290 -|0a116163 6d65436f 2f636f6c 6c656374| ..acmeCo/collect 000002a0 -|696f6e1a 082f6b65 792f6f6e 651a082f| ion../key/one../ 000002b0 -|6b65792f 74776f22 0b2f5f6d 6574612f| key/two"./_meta/ 000002c0 -|75756964 2a047479 70652a06 72656769| uuid*.type*.regi 000002d0 -|6f6e326a 0a092f6a 736f6e2f 70747212| on2j../json/ptr. 000002e0 -|07612d66 69656c64 20013252 0a07696e| .a-field .2R..in 000002f0 -|74656765 720a0673 7472696e 671a131a| teger..string... 00000300 -|03747970 22046461 746530b9 603a0365| .typ".date0.`:.e 00000310 -|6e632205 7469746c 652a0464 65736332| nc".title*.desc2 00000320 -|0e7b2264 6566223a 2261756c 74227d40| .{"def":"ault"}@ 00000330 -|014a0b08 01110000 00000040 8fc03a0e| .J.........@..:. 00000340 -|7b226163 6b223a22 74727565 227d4212| {"ack":"true"}B. 00000350 -|7b227772 69746522 3a227363 68656d61| {"write":"schema 00000360 -|227d4aa3 010a1270 61727469 74696f6e| "}J....partition 00000370 -|2f74656d 706c6174 6510031a 400a1e0a| /template...@... 00000380 -|0f657374 75617279 2e646576 2f666f6f| .estuary.dev/foo 00000390 -|120b6c61 62656c2d 76616c75 650a1e0a| ..label-value... 000003a0 -|0f657374 75617279 2e646576 2f626172| .estuary.dev/bar 000003b0 -|120b6f74 6865722d 76616c75 65224208| ..other-value"B. 000003c0 -|e9ec0610 031a1273 333a2f2f 6275636b| .......s3://buck 000003d0 -|65742f70 72656669 78220308 ac023207| et/prefix"....2. 000003e0 -|083e1080 e59a773a 18506174 687b7b50| .>....w:.Path{{P 000003f0 -|6f737466 69782e54 656d706c 6174657d| ostfix.Template} 00000400 -|7d300438 cfb0f501 5a117b22 72656164| }0.8....Z.{"read 00000410 -|223a2273 6368656d 61227d22 550a076b| ":"schema"}"U..k 00000420 -|65792f6f 6e651207 76616c2f 74776f1a| ey/one..val/two. 00000430 -|0d666c6f 775f646f 63756d65 6e74221d| .flow_document". 00000440 -|0a07615f 6669656c 6412127b 22666965| ..a_field..{"fie 00000450 -|6c64223a 22636f6e 66696722 7d22130a| ld":"config"}".. 00000460 -|0b6f7468 65722f66 69656c64 12043432| .other/field..42 00000470 -|2e353a5b 0a400a1e 0a0f6573 74756172| .5:[.@....estuar 00000480 -|792e6465 762f666f 6f120b6c 6162656c| y.dev/foo..label 00000490 -|2d76616c 75650a1e 0a0f6573 74756172| -value....estuar 000004a0 -|792e6465 762f6261 72120b6f 74686572| y.dev/bar..other 000004b0 -|2d76616c 75651217 0a150a08 6d792d6c| -value......my-l 000004c0 -|6162656c 12077072 65666978 2f180142| abel..prefix/..B 000004d0 -|316d6174 65726961 6c697a65 2f61636d| 1materialize/acm 000004e0 -|65436f2f 6d617465 7269616c 697a6174| eCo/materializat 000004f0 -|696f6e2f 736f6d65 25323070 6174682e| ion/some%20path. 00000500 -|76314803 520608cb c8d6a606 5a060880| v1H.R.......Z... 00000510 -|888ba106 60016a0e 736f6d65 25323070| ....`.j.some%20p 00000520 -|6174682e 76312a80 010a0e73 68617264| ath.v1*....shard 00000530 -|2f74656d 706c6174 651a0f72 65636f76| /template..recov 00000540 -|6572792f 70726566 6978220b 68696e74| ery/prefix".hint 00000550 -|2f707265 66697828 03320208 3c480152| /prefix(.2......w: 000007d0 -|18506174 687b7b50 6f737466 69782e54| .Path{{Postfix.T 000007e0 -|656d706c 6174657d 7d300438 cfb0f501| emplate}}0.8.... 000007f0 -|5a117b22 72656164 223a2273 6368656d| Z.{"read":"schem 00000800 -|61227d22 550a076b 65792f6f 6e651207| a"}"U..key/one.. 00000810 -|76616c2f 74776f1a 0d666c6f 775f646f| val/two..flow_do 00000820 -|63756d65 6e74221d 0a07615f 6669656c| cument"...a_fiel 00000830 -|6412127b 22666965 6c64223a 22636f6e| d..{"field":"con 00000840 -|66696722 7d22130a 0b6f7468 65722f66| fig"}"...other/f 00000850 -|69656c64 12043432 2e353a5b 0a400a1e| ield..42.5:[.@.. 00000860 -|0a0f6573 74756172 792e6465 762f666f| ..estuary.dev/fo 00000870 -|6f120b6c 6162656c 2d76616c 75650a1e| o..label-value.. 00000880 -|0a0f6573 74756172 792e6465 762f6261| ..estuary.dev/ba 00000890 -|72120b6f 74686572 2d76616c 75651217| r..other-value.. 000008a0 -|0a150a08 6d792d6c 6162656c 12077072| ....my-label..pr 000008b0 -|65666978 2f180142 316d6174 65726961| efix/..B1materia 000008c0 -|6c697a65 2f61636d 65436f2f 6d617465| lize/acmeCo/mate 000008d0 -|7269616c 697a6174 696f6e2f 736f6d65| rialization/some 000008e0 -|25323070 6174682e 76314803 520608cb| %20path.v1H.R... 000008f0 -|c8d6a606 5a060880 888ba106 60016a0e| ....Z.......`.j. 00000900 -|736f6d65 25323070 6174682e 76312a80| some%20path.v1*. 00000910 -|010a0e73 68617264 2f74656d 706c6174| ...shard/templat 00000920 -|651a0f72 65636f76 6572792f 70726566| e..recovery/pref 00000930 -|6978220b 68696e74 2f707265 66697828| ix".hint/prefix( 00000940 -|03320208 3c480152 400a1e0a 0f657374| .2......w:.Path{ 000001d0 +|7b506f73 74666978 2e54656d 706c6174| {Postfix.Templat 000001e0 +|657d7d30 0438cfb0 f5015a11 7b227265| e}}0.8....Z.{"re 000001f0 +|6164223a 22736368 656d6122 7d1a1d0a| ad":"schema"}... 00000200 +|07615f66 69656c64 12127b22 6669656c| .a_field..{"fiel 00000210 +|64223a22 636f6e66 6967227d 1a130a0b| d":"config"}.... 00000220 +|6f746865 722f6669 656c6412 0434322e| other/field..42. 00000230 +|35200332 0b30303a 31313a32 323a3333| 5 .2.00:11:22:33 00000240 +|1afd070a e0070a16 61636d65 436f2f6d| ........acmeCo/m 00000250 +|61746572 69616c69 7a617469 6f6e1008| aterialization.. 00000260 +|1a1d7b22 6d617465 7269616c 697a6522| ..{"materialize" 00000270 +|3a7b2263 6f6e6669 67223a34 327d7d22| :{"config":42}}" 00000280 +|d4050a15 7b227265 736f7572 6365223a| ....{"resource": 00000290 +|22636f6e 66696722 7d120473 6f6d6512| "config"}..some. 000002a0 +|04706174 681aa303 0a116163 6d65436f| .path.....acmeCo 000002b0 +|2f636f6c 6c656374 696f6e1a 082f6b65| /collection../ke 000002c0 +|792f6f6e 651a082f 6b65792f 74776f22| y/one../key/two" 000002d0 +|0b2f5f6d 6574612f 75756964 2a047479| ./_meta/uuid*.ty 000002e0 +|70652a06 72656769 6f6e3281 010a092f| pe*.region2..../ 000002f0 +|6a736f6e 2f707472 1207612d 6669656c| json/ptr..a-fiel 00000300 +|64200132 690a0769 6e746567 65720a06| d .2i..integer.. 00000310 +|73747269 6e671a13 1a037479 70220464| string....typ".d 00000320 +|61746530 b9603a03 656e6322 05746974| ate0.`:.enc".tit 00000330 +|6c652a04 64657363 320e7b22 64656622| le*.desc2.{"def" 00000340 +|3a226175 6c74227d 40014a0b 08011100| :"ault"}@.J..... 00000350 +|00000000 408fc052 15080a10 01181422| ....@..R......." 00000360 +|046e756c 6c220769 6e746567 65723a0e| .null".integer:. 00000370 +|7b226163 6b223a22 74727565 227d4212| {"ack":"true"}B. 00000380 +|7b227772 69746522 3a227363 68656d61| {"write":"schema 00000390 +|227d4aa3 010a1270 61727469 74696f6e| "}J....partition 000003a0 +|2f74656d 706c6174 6510031a 400a1e0a| /template...@... 000003b0 +|0f657374 75617279 2e646576 2f666f6f| .estuary.dev/foo 000003c0 +|120b6c61 62656c2d 76616c75 650a1e0a| ..label-value... 000003d0 +|0f657374 75617279 2e646576 2f626172| .estuary.dev/bar 000003e0 +|120b6f74 6865722d 76616c75 65224208| ..other-value"B. 000003f0 +|e9ec0610 031a1273 333a2f2f 6275636b| .......s3://buck 00000400 +|65742f70 72656669 78220308 ac023207| et/prefix"....2. 00000410 +|083e1080 e59a773a 18506174 687b7b50| .>....w:.Path{{P 00000420 +|6f737466 69782e54 656d706c 6174657d| ostfix.Template} 00000430 +|7d300438 cfb0f501 5a117b22 72656164| }0.8....Z.{"read 00000440 +|223a2273 6368656d 61227d22 550a076b| ":"schema"}"U..k 00000450 +|65792f6f 6e651207 76616c2f 74776f1a| ey/one..val/two. 00000460 +|0d666c6f 775f646f 63756d65 6e74221d| .flow_document". 00000470 +|0a07615f 6669656c 6412127b 22666965| ..a_field..{"fie 00000480 +|6c64223a 22636f6e 66696722 7d22130a| ld":"config"}".. 00000490 +|0b6f7468 65722f66 69656c64 12043432| .other/field..42 000004a0 +|2e353a5b 0a400a1e 0a0f6573 74756172| .5:[.@....estuar 000004b0 +|792e6465 762f666f 6f120b6c 6162656c| y.dev/foo..label 000004c0 +|2d76616c 75650a1e 0a0f6573 74756172| -value....estuar 000004d0 +|792e6465 762f6261 72120b6f 74686572| y.dev/bar..other 000004e0 +|2d76616c 75651217 0a150a08 6d792d6c| -value......my-l 000004f0 +|6162656c 12077072 65666978 2f180142| abel..prefix/..B 00000500 +|316d6174 65726961 6c697a65 2f61636d| 1materialize/acm 00000510 +|65436f2f 6d617465 7269616c 697a6174| eCo/materializat 00000520 +|696f6e2f 736f6d65 25323070 6174682e| ion/some%20path. 00000530 +|76314803 520608cb c8d6a606 5a060880| v1H.R.......Z... 00000540 +|888ba106 60016a0e 736f6d65 25323070| ....`.j.some%20p 00000550 +|6174682e 76312a80 010a0e73 68617264| ath.v1*....shard 00000560 +|2f74656d 706c6174 651a0f72 65636f76| /template..recov 00000570 +|6572792f 70726566 6978220b 68696e74| ery/prefix".hint 00000580 +|2f707265 66697828 03320208 3c480152| /prefix(.2......w:.Path{{P 00000820 +|6f737466 69782e54 656d706c 6174657d| ostfix.Template} 00000830 +|7d300438 cfb0f501 5a117b22 72656164| }0.8....Z.{"read 00000840 +|223a2273 6368656d 61227d22 550a076b| ":"schema"}"U..k 00000850 +|65792f6f 6e651207 76616c2f 74776f1a| ey/one..val/two. 00000860 +|0d666c6f 775f646f 63756d65 6e74221d| .flow_document". 00000870 +|0a07615f 6669656c 6412127b 22666965| ..a_field..{"fie 00000880 +|6c64223a 22636f6e 66696722 7d22130a| ld":"config"}".. 00000890 +|0b6f7468 65722f66 69656c64 12043432| .other/field..42 000008a0 +|2e353a5b 0a400a1e 0a0f6573 74756172| .5:[.@....estuar 000008b0 +|792e6465 762f666f 6f120b6c 6162656c| y.dev/foo..label 000008c0 +|2d76616c 75650a1e 0a0f6573 74756172| -value....estuar 000008d0 +|792e6465 762f6261 72120b6f 74686572| y.dev/bar..other 000008e0 +|2d76616c 75651217 0a150a08 6d792d6c| -value......my-l 000008f0 +|6162656c 12077072 65666978 2f180142| abel..prefix/..B 00000900 +|316d6174 65726961 6c697a65 2f61636d| 1materialize/acm 00000910 +|65436f2f 6d617465 7269616c 697a6174| eCo/materializat 00000920 +|696f6e2f 736f6d65 25323070 6174682e| ion/some%20path. 00000930 +|76314803 520608cb c8d6a606 5a060880| v1H.R.......Z... 00000940 +|888ba106 60016a0e 736f6d65 25323070| ....`.j.some%20p 00000950 +|6174682e 76312a80 010a0e73 68617264| ath.v1*....shard 00000960 +|2f74656d 706c6174 651a0f72 65636f76| /template..recov 00000970 +|6572792f 70726566 6978220b 68696e74| ery/prefix".hint 00000980 +|2f707265 66697828 03320208 3c480152| /prefix(.2.. 0 { + for iNdEx := len(m.Items) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Items[iNdEx]) + copy(dAtA[i:], m.Items[iNdEx]) + i = encodeVarintFlow(dAtA, i, uint64(len(m.Items[iNdEx]))) + i-- + dAtA[i] = 0x22 + } + } + if m.MaxItems != 0 { + i = encodeVarintFlow(dAtA, i, uint64(m.MaxItems)) + i-- + dAtA[i] = 0x18 + } + if m.HasMaxItems { + i-- + if m.HasMaxItems { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + } + if m.MinItems != 0 { + i = encodeVarintFlow(dAtA, i, uint64(m.MinItems)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *NetworkPort) Marshal() (dAtA []byte, err error) { size := m.ProtoSize() dAtA = make([]byte, size) @@ -3367,20 +3495,20 @@ func (m *CollectionSpec_Derivation) MarshalToSizedBuffer(dAtA []byte) (int, erro dAtA[i] = 0x2a } if len(m.ShuffleKeyTypes) > 0 { - dAtA9 := make([]byte, len(m.ShuffleKeyTypes)*10) - var j8 int + dAtA10 := make([]byte, len(m.ShuffleKeyTypes)*10) + var j9 int for _, num := range m.ShuffleKeyTypes { for num >= 1<<7 { - dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) + dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j8++ + j9++ } - dAtA9[j8] = uint8(num) - j8++ + dAtA10[j9] = uint8(num) + j9++ } - i -= j8 - copy(dAtA[i:], dAtA9[:j8]) - i = encodeVarintFlow(dAtA, i, uint64(j8)) + i -= j9 + copy(dAtA[i:], dAtA10[:j9]) + i = encodeVarintFlow(dAtA, i, uint64(j9)) i-- dAtA[i] = 0x22 } @@ -5230,6 +5358,10 @@ func (m *Inference) ProtoSize() (n int) { l = m.Numeric.ProtoSize() n += 1 + l + sovFlow(uint64(l)) } + if m.Array != nil { + l = m.Array.ProtoSize() + n += 1 + l + sovFlow(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5287,6 +5419,33 @@ func (m *Inference_Numeric) ProtoSize() (n int) { return n } +func (m *Inference_Array) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.MinItems != 0 { + n += 1 + sovFlow(uint64(m.MinItems)) + } + if m.HasMaxItems { + n += 2 + } + if m.MaxItems != 0 { + n += 1 + sovFlow(uint64(m.MaxItems)) + } + if len(m.Items) > 0 { + for _, s := range m.Items { + l = len(s) + n += 1 + l + sovFlow(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *NetworkPort) ProtoSize() (n int) { if m == nil { return 0 @@ -6833,6 +6992,42 @@ func (m *Inference) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Array", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthFlow + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthFlow + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Array == nil { + m.Array = &Inference_Array{} + } + if err := m.Array.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipFlow(dAtA[iNdEx:]) @@ -7134,6 +7329,147 @@ func (m *Inference_Numeric) Unmarshal(dAtA []byte) error { } return nil } +func (m *Inference_Array) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Array: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Array: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MinItems", wireType) + } + m.MinItems = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MinItems |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HasMaxItems", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.HasMaxItems = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxItems", wireType) + } + m.MaxItems = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxItems |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Items", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFlow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthFlow + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthFlow + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Items = append(m.Items, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipFlow(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthFlow + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NetworkPort) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/go/protocols/flow/flow.proto b/go/protocols/flow/flow.proto index 07af7e7ad7..e8255fee1b 100644 --- a/go/protocols/flow/flow.proto +++ b/go/protocols/flow/flow.proto @@ -139,6 +139,23 @@ message Inference { double maximum = 4; } Numeric numeric = 9; + // Array type-specific inferences. Will be nil if types doesn't include + // "array", or if the specification was built prior to array inference + // existing in the protocol. + message Array { + // Minimum number of items the array must contain. + uint32 min_items = 1; + // True if there is an inferred maximum allowed number of items the array + // may contain, otherwise False. + bool has_max_items = 2; + // Maximum number of items the array may contain. + uint32 max_items = 3; + // The possible types of items contained in this array. + // Subset of ["null", "boolean", "object", "array", "integer", "numeric", + // "string"]. + repeated string items = 4; + } + Array array = 10; } message NetworkPort {