From fed309a3d95663612e513ea0f3d5c1c4a682a3b8 Mon Sep 17 00:00:00 2001 From: Vinh Le Date: Mon, 21 Oct 2024 17:52:09 -0500 Subject: [PATCH 1/8] WIP: PG 17 --- .github/workflows/ci.yml | 2 +- Readme.md | 32 ++++++++++++++++++-------------- docker/ci/setup.sh | 6 +++--- extension/Cargo.toml | 11 ++++++----- tools/build | 2 +- tools/dependencies.sh | 6 +++--- 6 files changed, 32 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0b7981d..5d3b219f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: fail-fast: false max-parallel: 12 matrix: - pgversion: [13, 14, 15, 16] + pgversion: [13, 14, 15, 16, 17] container: - os: rockylinux version: "9" diff --git a/Readme.md b/Readme.md index 64c63de6..ea7c662d 100644 --- a/Readme.md +++ b/Readme.md @@ -1,7 +1,6 @@ [![CI](https://github.com/timescale/timescaledb-toolkit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/timescale/timescaledb-toolkit/actions/workflows/ci.yml) - -# TimescaleDB Toolkit # +# TimescaleDB Toolkit This repository is the home of the TimescaleDB Toolkit team. Our mission is to ease all things analytics when using TimescaleDB, with a particular focus on @@ -14,16 +13,15 @@ Documentation for this version of the TimescaleDB Toolkit extension can be found in this repository at [`docs`](https://github.com/timescale/timescaledb-toolkit/tree/main/docs). The release history can be found on this repo's [GitHub releases](https://github.com/timescale/timescaledb-toolkit/releases). +## 🖥 Try It Out -## 🖥 Try It Out ## - -The extension comes pre-installed on all [Timescale Cloud](https://www.timescale.com/products#timescale-cloud) instances and also on our full-featured [`timescale/timescaledb-ha` docker image](https://hub.docker.com/r/timescale/timescaledb-ha). +The extension comes pre-installed on all [Timescale Cloud](https://www.timescale.com/products#timescale-cloud) instances and also on our full-featured [`timescale/timescaledb-ha` docker image](https://hub.docker.com/r/timescale/timescaledb-ha). If DEB and RPM packages are a better fit for your situation, refer to the [Install Toolkit on self-hosted TimescaleDB](https://docs.timescale.com/self-hosted/latest/tooling/install-toolkit/#install-toolkit-on-self-hosted-timescaledb) how-to guide for further instructions on installing the extension via your package manager. All versions of the extension contain experimental features in the `toolkit_experimental` schema. See [our docs section on experimental features](/docs/README.md#tag-notes) for more details. -## 💿 Installing From Source ## +## 💿 Installing From Source ### Supported platforms @@ -34,22 +32,26 @@ The engineering team regularly tests the extension on the following platforms: - x86_64-apple-darwin (MacOS 12) (tested frequently on eng workstation) - aarch64-apple-darwin (MacOS 12) (tested frequently on eng workstation) -As for other platforms: patches welcome! +As for other platforms: patches welcome! -### 🔧 Tools Setup ### +### 🔧 Tools Setup Building the extension requires valid [rust](https://www.rust-lang.org/) (we build and test on 1.65), [rustfmt](https://github.com/rust-lang/rustfmt), and clang installs, along with the postgres headers for whichever version of postgres you are running, and pgrx. We recommend installing rust using the [official instructions](https://www.rust-lang.org/tools/install): + ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` + and build tools, the postgres headers, in the preferred manner for your system. You may also need to install OpenSSl. For Ubuntu you can follow the [postgres install instructions](https://www.postgresql.org/download/linux/ubuntu/) then run + ```bash sudo apt-get install make gcc pkg-config clang postgresql-server-dev-14 libssl-dev ``` Next you need [cargo-pgrx](https://github.com/tcdi/pgrx), which can be installed with + ```bash cargo install --version '=0.10.2' --force cargo-pgrx ``` @@ -62,28 +64,31 @@ Finally, setup the pgrx development environment with cargo pgrx init --pg14 pg_config ``` - Installing from source is also available on macOS and requires the same set of prerequisites and set up commands listed above. -### 💾 Building and Installing the extension ### +### 💾 Building and Installing the extension Download or clone this repository, and switch to the `extension` subdirectory, e.g. + ```bash git clone https://github.com/timescale/timescaledb-toolkit && \ cd timescaledb-toolkit/extension ``` + Then run + ``` cargo pgrx install --release && \ cargo run --manifest-path ../tools/post-install/Cargo.toml -- pg_config ``` To initialize the extension after installation, enter the following into `psql`: + ``` CREATE EXTENSION timescaledb_toolkit; ``` -## ✏️ Get Involved ## +## ✏️ Get Involved The TimescaleDB Toolkit project is still in the initial planning stage as we decide our priorities and what to implement first. As such, now is a great time @@ -92,17 +97,16 @@ to help shape the project's direction! Have a look at the and feel free to comment on the features, expand the list, or hop on the [Discussions forum](https://github.com/timescale/timescaledb-toolkit/discussions) for more in-depth discussions. -### 🔨 Testing ### +### 🔨 Testing See above for prerequisites and installation instructions. -You can run tests against a postgres version `pg12`, `pg13`, `pg14`, `pg15` or `pg16` using +You can run tests against a postgres version `pg12`, `pg13`, `pg14`, `pg15`, `pg16` or `pg17` using ``` cargo pgrx test ${postgres_version} ``` - ## 🐯 About TimescaleDB **[TimescaleDB](https://github.com/timescale/timescaledb)** is a diff --git a/docker/ci/setup.sh b/docker/ci/setup.sh index 0f8345cb..07a059f4 100755 --- a/docker/ci/setup.sh +++ b/docker/ci/setup.sh @@ -54,8 +54,8 @@ if $privileged; then centos | rockylinux) case $OS_VERSION in 7) - export PG_VERSIONS="13 14 15" - export TSDB_PG_VERSIONS="13 14 15" + export PG_VERSIONS="13 14 15 16 17" + export TSDB_PG_VERSIONS="13 14 15 16 17" # Postgresql packages require both # - llvm-toolset-7-clang from centos-release-scl-rh # - llvm5.0-devel from epel-release @@ -151,7 +151,7 @@ EOF debian | ubuntu) # TimescaleDB does not have packages for Debian 12 if [ $OS_VERSION -ge 12 ]; then - export TSDB_PG_VERSIONS="13 14 15" + export TSDB_PG_VERSIONS="13 14 15 16 17" fi # Image comes in with no package lists so we have to start with this. diff --git a/extension/Cargo.toml b/extension/Cargo.toml index d6e50888..ea62d132 100644 --- a/extension/Cargo.toml +++ b/extension/Cargo.toml @@ -7,20 +7,21 @@ edition = "2021" crate-type = ["cdylib"] [features] -default = ["pg15"] +default = ["pg16"] pg12 = ["pgrx/pg12", "pgrx-tests/pg12"] pg13 = ["pgrx/pg13", "pgrx-tests/pg13"] pg14 = ["pgrx/pg14", "pgrx-tests/pg14"] pg15 = ["pgrx/pg15", "pgrx-tests/pg15"] pg16 = ["pgrx/pg16", "pgrx-tests/pg16"] +pg17 = ["pgrx/pg17", "pgrx-tests/pg17"] pg_test = ["approx"] [dependencies] # Keep synchronized with `cargo install --version N.N.N cargo-pgrx` in Readme.md and docker/ci/Dockerfile # Also `pgrx-tests` down below in `dev-dependencies`. -pgrx = "=0.10.2" -pgrx-macros = "=0.10.2" -pgrx-sql-entity-graph = "=0.10.2" +pgrx = "=0.12.16" +pgrx-macros = "=0.12.16" +pgrx-sql-entity-graph = "=0.12.16" encodings = {path="../crates/encodings"} flat_serialize = {path="../crates/flat_serialize/flat_serialize"} flat_serialize_macro = {path="../crates/flat_serialize/flat_serialize_macro"} @@ -57,5 +58,5 @@ spfunc = "0.1.0" statrs = "0.15.0" [dev-dependencies] -pgrx-tests = "=0.10.2" +pgrx-tests = "=0.12.16" approx = "0.4.0" diff --git a/tools/build b/tools/build index 1c302b05..0f5c524f 100755 --- a/tools/build +++ b/tools/build @@ -20,7 +20,7 @@ usage() { } require_pg_version() { - [ -n "$pg_version" ] || die 'specify one of -pg12 | -pg13 | -pg14 | -pg15 | -pg16' + [ -n "$pg_version" ] || die 'specify one of -pg12 | -pg13 | -pg14 | -pg15 | -pg16 | -pg17' } find_pg_config() { diff --git a/tools/dependencies.sh b/tools/dependencies.sh index 98fe986c..399b0385 100644 --- a/tools/dependencies.sh +++ b/tools/dependencies.sh @@ -7,14 +7,14 @@ # All our automation scripts read this, so at least we're not duplicating this # information across all those. -PG_VERSIONS='12 13 14 15 16' +PG_VERSIONS='12 13 14 15 16 17' # TODO: extend this with 16 this once TimescaleDB supports PostgreSQL 16: issue #5752 -TSDB_PG_VERSIONS='12 13 14 15' +TSDB_PG_VERSIONS='12 13 14 15 16 17' CARGO_EDIT=0.11.2 # Keep synchronized with extension/Cargo.toml and `cargo install --version N.N.N cargo-pgrx` in Readme.md . -PGRX_VERSION=0.10.2 +PGRX_VERSION=0.12.16 RUST_TOOLCHAIN=1.70.0 RUST_PROFILE=minimal From 562765b718b2c4ab4e5001043f5ff5da2a778199 Mon Sep 17 00:00:00 2001 From: Vinh Le Date: Mon, 21 Oct 2024 17:56:05 -0500 Subject: [PATCH 2/8] typo on pgrx version --- extension/Cargo.toml | 8 ++++---- tools/dependencies.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extension/Cargo.toml b/extension/Cargo.toml index ea62d132..2b49c36b 100644 --- a/extension/Cargo.toml +++ b/extension/Cargo.toml @@ -19,9 +19,9 @@ pg_test = ["approx"] [dependencies] # Keep synchronized with `cargo install --version N.N.N cargo-pgrx` in Readme.md and docker/ci/Dockerfile # Also `pgrx-tests` down below in `dev-dependencies`. -pgrx = "=0.12.16" -pgrx-macros = "=0.12.16" -pgrx-sql-entity-graph = "=0.12.16" +pgrx = "=0.12.6" +pgrx-macros = "=0.12.6" +pgrx-sql-entity-graph = "=0.12.6" encodings = {path="../crates/encodings"} flat_serialize = {path="../crates/flat_serialize/flat_serialize"} flat_serialize_macro = {path="../crates/flat_serialize/flat_serialize_macro"} @@ -58,5 +58,5 @@ spfunc = "0.1.0" statrs = "0.15.0" [dev-dependencies] -pgrx-tests = "=0.12.16" +pgrx-tests = "=0.12.6" approx = "0.4.0" diff --git a/tools/dependencies.sh b/tools/dependencies.sh index 399b0385..dec75499 100644 --- a/tools/dependencies.sh +++ b/tools/dependencies.sh @@ -14,7 +14,7 @@ TSDB_PG_VERSIONS='12 13 14 15 16 17' CARGO_EDIT=0.11.2 # Keep synchronized with extension/Cargo.toml and `cargo install --version N.N.N cargo-pgrx` in Readme.md . -PGRX_VERSION=0.12.16 +PGRX_VERSION=0.12.6 RUST_TOOLCHAIN=1.70.0 RUST_PROFILE=minimal From a76e43c9d865feedfd3e584c4ca1a98ebe67d159 Mon Sep 17 00:00:00 2001 From: Vinh Date: Tue, 22 Oct 2024 08:34:48 -0500 Subject: [PATCH 3/8] Update Readme.md Co-authored-by: Sebastian Webber Signed-off-by: Vinh --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index ea7c662d..cf68b44d 100644 --- a/Readme.md +++ b/Readme.md @@ -53,7 +53,7 @@ sudo apt-get install make gcc pkg-config clang postgresql-server-dev-14 libssl-d Next you need [cargo-pgrx](https://github.com/tcdi/pgrx), which can be installed with ```bash -cargo install --version '=0.10.2' --force cargo-pgrx +cargo install --version '=0.12.6' --force cargo-pgrx ``` You must reinstall cargo-pgrx whenever you update your Rust compiler, since cargo-pgrx needs to be built with the same compiler as Toolkit. From 097c1c1af9384e1ab64800b246c5ac9e38cb740c Mon Sep 17 00:00:00 2001 From: Vinh Le Date: Tue, 22 Oct 2024 09:51:23 -0500 Subject: [PATCH 4/8] Add PG 16 and PG 17 to bors.toml --- bors.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bors.toml b/bors.toml index d09104f2..0143e8b5 100644 --- a/bors.toml +++ b/bors.toml @@ -12,6 +12,12 @@ status = [ "Test Postgres (15, debian, 11, debian-11-amd64)", "Test Postgres (15, rockylinux, 9, rockylinux-9-x86_64)", "Test Postgres (15, centos, 7, centos-7-x86_64)", + "Test Postgres (16, debian, 11, debian-11-amd64)", + "Test Postgres (16, rockylinux, 9, rockylinux-9-x86_64)", + "Test Postgres (16, centos, 7, centos-7-x86_64)", + "Test Postgres (17, debian, 11, debian-11-amd64)", + "Test Postgres (17, rockylinux, 9, rockylinux-9-x86_64)", + "Test Postgres (17, centos, 7, centos-7-x86_64)", ] delete-merged-branches = true timeout_sec = 3600 # 60 min From 9990b6d85e23ca939f63d560800103cd6164420c Mon Sep 17 00:00:00 2001 From: Vinh Le Date: Tue, 22 Oct 2024 11:01:34 -0500 Subject: [PATCH 5/8] Set RUST_TOOLCHAIN to 1.76.0 --- tools/dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dependencies.sh b/tools/dependencies.sh index dec75499..d792ae30 100644 --- a/tools/dependencies.sh +++ b/tools/dependencies.sh @@ -16,7 +16,7 @@ CARGO_EDIT=0.11.2 # Keep synchronized with extension/Cargo.toml and `cargo install --version N.N.N cargo-pgrx` in Readme.md . PGRX_VERSION=0.12.6 -RUST_TOOLCHAIN=1.70.0 +RUST_TOOLCHAIN=1.76.0 RUST_PROFILE=minimal RUST_COMPONENTS=clippy,rustfmt From cdcc660660cba955f14e258eee4f690835310b36 Mon Sep 17 00:00:00 2001 From: Vinh Le Date: Tue, 22 Oct 2024 15:43:43 -0500 Subject: [PATCH 6/8] Attempt to abide and clean up workflows for unsupported PG versions --- .github/workflows/ci.yml | 17 +++++++++++++---- bors.toml | 4 ---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d3b219f..9383a7b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: fail-fast: false max-parallel: 12 matrix: - pgversion: [13, 14, 15, 16, 17] + pgversion: [14, 15, 16, 17] container: - os: rockylinux version: "9" @@ -69,7 +69,6 @@ jobs: version: "22.04" image: ubuntu-22.04-amd64 schedule: ${{ inputs.all-platforms || ( github.event_name == 'schedule' && github.event.schedule == '0 6 * * 1-4' ) }} - exclude: - container: skip: true @@ -80,17 +79,27 @@ jobs: os: centos version: "7" pgversion: 16 + include: # TimescaleDB as of 2.12.0 no longer supports PostgreSQL 12. # To allow us to do run CI against PostgreSQL 12, we therefore explicitly # remove all CI's, except if we are run against the latest supported TimescaleDB # version for PostgreSQL 12, which is 2.11.2 - include: - pgversion: 12 container: image: debian-11-amd64 os: debian version: "11" tsdb_commit: 2.11.2 + # TimescaleDB as of 2.16.0 no longer supports PostgreSQL 13. + # To allow us to do run CI against PostgreSQL 13, we therefore explicitly + # remove all CI's, except if we are run against the latest supported TimescaleDB + # version for PostgreSQL 13, which is 2.15.3 + - pgversion: 13 + container: + image: debian-11-amd64 + os: debian + version: "11" + tsdb_commit: 2.15.3 env: # TODO Why? Cargo default is to pass `-C incremental` to rustc; why don't we want that? # https://doc.rust-lang.org/rustc/codegen-options/index.html#incremental @@ -124,7 +133,7 @@ jobs: - name: Build and install TimescaleDB if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 8 * * 1-4') || inputs.tsdb-commit != '' }} - run: ./tools/install-timescaledb '${{ matrix.pgversion }}' '${{ matrix.tsdb_commit || inputs.tsdb-repo || 'https://github.com/timescale/timescaledb.git' }}' '${{ inputs.tsdb-commit == '' && 'main' || matrix.tsdb_commit || inputs.tsdb-commit }}' + run: ./tools/install-timescaledb '${{ matrix.pgversion }}' '${{ inputs.tsdb-repo || 'https://github.com/timescale/timescaledb.git' }}' '${{ inputs.tsdb-commit == '' && 'main' || matrix.tsdb_commit || inputs.tsdb-commit }}' # TODO After the container image contains a primed target dir, is this still worth it? # Only possible advantage is this one is per-pg-version but what's the impact? diff --git a/bors.toml b/bors.toml index 0143e8b5..c959a4f4 100644 --- a/bors.toml +++ b/bors.toml @@ -1,11 +1,7 @@ status = [ "Clippy/rustfmt Test", "Test Postgres (12, debian, 11, debian-11-amd64)", - "Test Postgres (12, rockylinux, 9, rockylinux-9-x86_64)", - "Test Postgres (12, centos, 7, centos-7-x86_64)", "Test Postgres (13, debian, 11, debian-11-amd64)", - "Test Postgres (13, rockylinux, 9, rockylinux-9-x86_64)", - "Test Postgres (13, centos, 7, centos-7-x86_64)", "Test Postgres (14, debian, 11, debian-11-amd64)", "Test Postgres (14, rockylinux, 9, rockylinux-9-x86_64)", "Test Postgres (14, centos, 7, centos-7-x86_64)", From 734dd00d7e87cd6de4e386f2aec93e18e47e616b Mon Sep 17 00:00:00 2001 From: Brent Graveland Date: Wed, 23 Oct 2024 10:47:01 -0600 Subject: [PATCH 7/8] Update to rust 1.82.0 --- tools/build | 2 ++ tools/dependencies.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/build b/tools/build index 0f5c524f..ee2988b4 100755 --- a/tools/build +++ b/tools/build @@ -45,6 +45,8 @@ find_profile() { [ $# -ge 1 ] || usage +rustc --version + while [ $# -gt 0 ]; do arg="$1" shift diff --git a/tools/dependencies.sh b/tools/dependencies.sh index d792ae30..f9e23498 100644 --- a/tools/dependencies.sh +++ b/tools/dependencies.sh @@ -16,7 +16,7 @@ CARGO_EDIT=0.11.2 # Keep synchronized with extension/Cargo.toml and `cargo install --version N.N.N cargo-pgrx` in Readme.md . PGRX_VERSION=0.12.6 -RUST_TOOLCHAIN=1.76.0 +RUST_TOOLCHAIN=1.82.0 RUST_PROFILE=minimal RUST_COMPONENTS=clippy,rustfmt From 80476af033f851236a19bc656aaa0a067691f514 Mon Sep 17 00:00:00 2001 From: Brent Graveland Date: Wed, 23 Oct 2024 12:34:46 -0600 Subject: [PATCH 8/8] Use the timescaledev/toolkit-builder-test container This container is built in CI with the current release, so use that one when testing. --- .github/workflows/ci.yml | 2 +- .github/workflows/clippy_rustfmt.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9383a7b4..9937ecb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: name: Test Postgres runs-on: ubuntu-20.04 container: - image: ${{ inputs.container-image || 'timescaledev/toolkit-builder' }}:${{ matrix.container.image }} + image: ${{ inputs.container-image || 'timescaledev/toolkit-builder-test' }}:${{ matrix.container.image }} strategy: fail-fast: false max-parallel: 12 diff --git a/.github/workflows/clippy_rustfmt.yml b/.github/workflows/clippy_rustfmt.yml index c4817c44..43125aa0 100644 --- a/.github/workflows/clippy_rustfmt.yml +++ b/.github/workflows/clippy_rustfmt.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-20.04 container: # Duplicated from ci.yml - image: ${{ inputs.container-image || 'timescaledev/toolkit-builder:debian-11-amd64' }} + image: ${{ inputs.container-image || 'timescaledev/toolkit-builder-test:debian-11-amd64' }} env: # TODO: See TODOs on duplicate block in ci.yml CARGO_INCREMENTAL: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96a9c307..acea07bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: name: Release runs-on: ubuntu-20.04 container: - image: timescaledev/toolkit-builder:debian-11-amd64 + image: timescaledev/toolkit-builder-test:debian-11-amd64 steps: - name: Checkout Repository