Skip to content

Commit

Permalink
chore: update core deps
Browse files Browse the repository at this point in the history 
- arrow/parquet deps are patched (as in core)
- two specific code changes to cope with change in API
  - TransitionPartitionId
  - arrow buffers can take &[u8] directly without `to_vec()`/`vec!`
  • Loading branch information
@praveen-influx
praveen-influx committed Nov 11, 2024
1 parent 35e29d1 commit 1a61b75
Show file tree
Hide file tree
Showing 8 changed files with 912 additions and 695 deletions.
1,402 changes: 792 additions & 610 deletions Cargo.lock
View file

Large diffs are not rendered by default.

104 changes: 62 additions & 42 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,28 +37,28 @@ license = "MIT OR Apache-2.0"

[workspace.dependencies]
anyhow = "1.0"
arrow = { version = "52.1.0", features = ["prettyprint", "chrono-tz"] }
arrow-array = "52.2.0"
arrow-buffer = "52.2.0"
arrow-csv = "52.2.0"
arrow-flight = { version = "52.2.0", features = ["flight-sql-experimental"] }
arrow-json = "52.2.0"
arrow-schema = "52.2.0"
arrow = { version = "53.0.0", features = ["prettyprint", "chrono-tz"] }
arrow-array = "53.0.0"
arrow-buffer = "53.0.0"
arrow-csv = "53.0.0"
arrow-flight = { version = "53.0.0", features = ["flight-sql-experimental"] }
arrow-json = "53.0.0"
arrow-schema = "53.0.0"
assert_cmd = "2.0.14"
async-trait = "0.1"
backtrace = "0.3"
base64 = "0.22.0"
bimap = "0.6.3"
byteorder = "1.3.4"
bytes = "1.5"
bytes = "1.8"
chrono = "0.4"
clap = { version = "4", features = ["derive", "env", "string"] }
clru = "0.6.2"
crc32fast = "1.2.0"
crossbeam-channel = "0.5.11"
csv = "1.3.0"
datafusion = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "5de0c3577fd30dcf9213f428222a29efae789807" }
datafusion-proto = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "5de0c3577fd30dcf9213f428222a29efae789807" }
datafusion = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "c27d5f2356a21ee6224c149ee971c89c2cc13e18" }
datafusion-proto = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "c27d5f2356a21ee6224c149ee971c89c2cc13e18" }
dashmap = "6.1.0"
dotenvy = "0.15.7"
flate2 = "1.0.27"
Expand All @@ -76,9 +76,9 @@ mime = "0.3.17"
mockito = { version = "1.4.0", default-features = false }
mockall = { version = "0.13.0" }
num_cpus = "1.16.0"
object_store = "0.10.2"
object_store = "0.11.1"
parking_lot = "0.12.1"
parquet = { version = "52.2.0", features = ["object_store"] }
parquet = { version = "53.0.0", features = ["object_store"] }
pbjson = "0.6.0"
pbjson-build = "0.6.2"
pbjson-types = "0.6.0"
Expand Down Expand Up @@ -117,36 +117,36 @@ uuid = { version = "1", features = ["v4"] }
num = { version = "0.4.3" }

# Core.git crates we depend on
arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7"}
authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7", features = ["http"] }
clap_blocks = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7", features = ["v3"] }
influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_catalog = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7", features = ["v3"] }
service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7" }
trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "1eaa4ed5ea147bc24db98d9686e457c124dfd5b7", default-features = true, features = ["clap"] }
arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
clap_blocks = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_catalog = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }
trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "6fcbb004232738d55655f32f4ad2385523d10696" }

[workspace.lints.rust]
missing_copy_implementations = "deny"
Expand Down Expand Up @@ -212,3 +212,23 @@ opt-level = 3
# arrow-data = { path = "../arrow-rs/arrow-data" }
# arrow-buffer = { path = "../arrow-rs/arrow-buffer" }
# arrow-ipc = { path = "../arrow-rs/arrow-ipc" }

## NB: This is taken from Iox
## Use patch of arrow-rs with an older version of tonic
## until we have upgraded hyper: https://github.com/influxdata/influxdb_iox/issues/9340
## see https://github.com/influxdata/arrow-rs/pull/3
[patch.crates-io]
arrow = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-array = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-buffer = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-cast = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-csv = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-data = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-ipc = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-json = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-schema = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-select = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-string = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-ord = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
arrow-flight = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
parquet = { git = "https://github.com/influxdata/arrow-rs.git", rev = "e38787d2177f2ebfa481bfac62d208eef8ea82fb" }
2 changes: 1 addition & 1 deletion influxdb3_catalog/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ license.workspace = true
# Core Crates
influxdb-line-protocol.workspace = true
observability_deps.workspace = true
schema.workspace = true
schema = { workspace = true, features = ["v3"] }

# Local deps
influxdb3_id = { path = "../influxdb3_id" }
Expand Down
6 changes: 3 additions & 3 deletions influxdb3_server/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,16 +345,16 @@ mod tests {

assert_eq!(
batches[0]["host"].to_data().child_data()[0].buffers()[1],
Buffer::from([b'a'].to_vec())
Buffer::from([b'a'])
);

assert_eq!(
batches[0]["time"].to_data().buffers(),
&[Buffer::from(vec![123, 0, 0, 0, 0, 0, 0, 0])]
&[Buffer::from([123, 0, 0, 0, 0, 0, 0, 0])]
);
assert_eq!(
batches[0]["val"].to_data().buffers(),
&[Buffer::from(1_u64.to_le_bytes().to_vec())]
&[Buffer::from(1_u64.to_le_bytes())]
);

shutdown.cancel();
Expand Down
54 changes: 27 additions & 27 deletions influxdb3_server/src/query_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use iox_query::query_log::QueryLog;
use iox_query::query_log::QueryText;
use iox_query::query_log::StateReceived;
use iox_query::query_log::{QueryCompletedToken, QueryLogEntries};
use iox_query::{Extension, QueryDatabase};
use iox_query::QueryDatabase;
use iox_query::{QueryChunk, QueryNamespace};
use iox_query_influxql::frontend::planner::InfluxQLQueryPlanner;
use iox_query_params::StatementParams;
Expand Down Expand Up @@ -405,23 +405,23 @@ impl Database {

#[async_trait]
impl QueryNamespace for Database {
async fn chunks(
&self,
table_name: &str,
filters: &[Expr],
projection: Option<&Vec<usize>>,
ctx: IOxSessionContext,
) -> Result<Vec<Arc<dyn QueryChunk>>, DataFusionError> {
let _span_recorder = SpanRecorder::new(ctx.child_span("QueryDatabase::chunks"));
debug!(%table_name, ?filters, "Database as QueryNamespace::chunks");

let Some(table) = self.query_table(table_name).await else {
debug!(%table_name, "No entry for table");
return Ok(vec![]);
};

table.chunks(&ctx.inner().state(), projection, filters, None)
}
// async fn chunks(
// &self,
// table_name: &str,
// filters: &[Expr],
// projection: Option<&Vec<usize>>,
// ctx: IOxSessionContext,
// ) -> Result<Vec<Arc<dyn QueryChunk>>, DataFusionError> {
// let _span_recorder = SpanRecorder::new(ctx.child_span("QueryDatabase::chunks"));
// debug!(%table_name, ?filters, "Database as QueryNamespace::chunks");
//
// let Some(table) = self.query_table(table_name).await else {
// debug!(%table_name, "No entry for table");
// return Ok(vec![]);
// };
//
// table.chunks(&ctx.inner().state(), projection, filters, None)
// }

fn retention_time_ns(&self) -> Option<i64> {
None
Expand Down Expand Up @@ -474,7 +474,7 @@ impl QueryNamespace for Database {

fn new_extended_query_context(
&self,
_extension: Arc<dyn Extension>,
_extension: std::option::Option<std::sync::Arc<(dyn iox_query::Extension + 'static)>>,
_span_ctx: Option<SpanContext>,
_query_config: Option<&QueryConfig>,
) -> IOxSessionContext {
Expand Down Expand Up @@ -750,10 +750,10 @@ mod tests {
"+------------+------------+-----------+----------+----------+",
"| table_name | size_bytes | row_count | min_time | max_time |",
"+------------+------------+-----------+----------+----------+",
"| cpu | 2142 | 2 | 0 | 10 |",
"| cpu | 2142 | 2 | 20 | 30 |",
"| cpu | 2142 | 2 | 40 | 50 |",
"| cpu | 2142 | 2 | 60 | 70 |",
"| cpu | 1940 | 2 | 0 | 10 |",
"| cpu | 1940 | 2 | 20 | 30 |",
"| cpu | 1940 | 2 | 40 | 50 |",
"| cpu | 1940 | 2 | 60 | 70 |",
"+------------+------------+-----------+----------+----------+",
],
},
Expand All @@ -763,10 +763,10 @@ mod tests {
"+------------+------------+-----------+----------+----------+",
"| table_name | size_bytes | row_count | min_time | max_time |",
"+------------+------------+-----------+----------+----------+",
"| mem | 2142 | 2 | 0 | 10 |",
"| mem | 2142 | 2 | 20 | 30 |",
"| mem | 2142 | 2 | 40 | 50 |",
"| mem | 2142 | 2 | 60 | 70 |",
"| mem | 1940 | 2 | 0 | 10 |",
"| mem | 1940 | 2 | 20 | 30 |",
"| mem | 1940 | 2 | 40 | 50 |",
"| mem | 1940 | 2 | 60 | 70 |",
"+------------+------------+-----------+----------+----------+",
],
},
Expand Down
2 changes: 1 addition & 1 deletion influxdb3_wal/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ license.workspace = true
# Core crates
data_types.workspace = true
iox_time.workspace = true
influxdb-line-protocol.workspace = true
influxdb-line-protocol = { workspace = true, features = ["v3"] }
observability_deps.workspace = true
schema.workspace = true

Expand Down
14 changes: 10 additions & 4 deletions influxdb3_write/src/write_buffer/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ use crate::{
PersistedSnapshot, Precision, WriteBuffer, WriteLineError,
};
use async_trait::async_trait;
use data_types::{ChunkId, ChunkOrder, ColumnType, NamespaceName, NamespaceNameError};
use data_types::{
ChunkId, ChunkOrder, ColumnType, NamespaceName, NamespaceNameError, PartitionHashId,
PartitionId,
};
use datafusion::catalog::Session;
use datafusion::common::DataFusionError;
use datafusion::datasource::object_store::ObjectStoreUrl;
Expand Down Expand Up @@ -352,9 +355,12 @@ pub fn parquet_chunk_from_file(
chunk_order: i64,
) -> ParquetChunk {
let partition_key = data_types::PartitionKey::from(parquet_file.chunk_time.to_string());
let partition_id = data_types::partition::TransitionPartitionId::new(
data_types::TableId::new(0),
&partition_key,
let partition_id = data_types::partition::TransitionPartitionId::from_parts(
PartitionId::new(0),
Some(PartitionHashId::new(
data_types::TableId::new(0),
&partition_key,
)),
);

let chunk_stats = create_chunk_statistics(
Expand Down
Loading

0 comments on commit 1a61b75

Please sign in to comment.