Merge branch 'master' into dependabot/go_modules/go-dependencies-27ee…

…bd2a4b

Signed-off-by: Charlie Le <[email protected]>

.github/workflows/test-build-deploy.yml

@@ -189,6 +189,11 @@ jobs:
             docker pull quay.io/cortexproject/cortex:v1.17.0
             docker pull quay.io/cortexproject/cortex:v1.17.1
             docker pull quay.io/cortexproject/cortex:v1.18.0
+            docker pull quay.io/cortexproject/cortex:v1.18.1
+          elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then
+            docker pull quay.io/cortexproject/cortex:v1.18.1 
+            docker pull quay.io/prometheus/prometheus:v2.51.0
+            docker pull quay.io/prometheus/prometheus:v2.55.0
           fi
           docker pull memcached:1.6.1
           docker pull redis:7.0.4-alpine

CHANGELOG.md

@@ -9,7 +9,10 @@
 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
 * [FEATURE] Store Gateway: Add an in-memory chunk cache. #6245
 * [FEATURE] Chunk Cache: Support multi level cache and add metrics. #6249
+* [ENHANCEMENT] S3 Bucket Client: Add a list objects version configs to configure list api object version. #6280
+* [ENHANCEMENT] OpenStack Swift: Add application credential configs for Openstack swift object storage backend. #6255
 * [ENHANCEMENT] Query Frontend: Add new query stats metrics `cortex_query_samples_scanned_total` and `cortex_query_peak_samples` to track scannedSamples and peakSample per user. #6228
+* [ENHANCEMENT] Ingester: Add option `ingester.disable-chunk-trimming` to disable chunk trimming. #6300
 * [ENHANCEMENT] Ingester: Add `blocks-storage.tsdb.wal-compression-type` to support zstd wal compression type. #6232
 * [ENHANCEMENT] Query Frontend: Add info field to query response. #6207
 * [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188

docs/blocks-storage/querier.md

@@ -312,6 +312,10 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.send-content-md5
     [send_content_md5: <boolean> | default = true]
 
+    # The list api version. Supported values are: v1, v2, and ''.
+    # CLI flag: -blocks-storage.s3.list-objects-version
+    [list_objects_version: <string> | default = ""]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]
@@ -452,6 +456,18 @@ blocks_storage:
     # CLI flag: -blocks-storage.swift.auth-url
     [auth_url: <string> | default = ""]
 
+    # OpenStack Swift application credential ID.
+    # CLI flag: -blocks-storage.swift.application-credential-id
+    [application_credential_id: <string> | default = ""]
+
+    # OpenStack Swift application credential name.
+    # CLI flag: -blocks-storage.swift.application-credential-name
+    [application_credential_name: <string> | default = ""]
+
+    # OpenStack Swift application credential secret.
+    # CLI flag: -blocks-storage.swift.application-credential-secret
+    [application_credential_secret: <string> | default = ""]
+
     # OpenStack Swift username.
     # CLI flag: -blocks-storage.swift.username
     [username: <string> | default = ""]

docs/blocks-storage/store-gateway.md

@@ -403,6 +403,10 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.send-content-md5
     [send_content_md5: <boolean> | default = true]
 
+    # The list api version. Supported values are: v1, v2, and ''.
+    # CLI flag: -blocks-storage.s3.list-objects-version
+    [list_objects_version: <string> | default = ""]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]
@@ -543,6 +547,18 @@ blocks_storage:
     # CLI flag: -blocks-storage.swift.auth-url
     [auth_url: <string> | default = ""]
 
+    # OpenStack Swift application credential ID.
+    # CLI flag: -blocks-storage.swift.application-credential-id
+    [application_credential_id: <string> | default = ""]
+
+    # OpenStack Swift application credential name.
+    # CLI flag: -blocks-storage.swift.application-credential-name
+    [application_credential_name: <string> | default = ""]
+
+    # OpenStack Swift application credential secret.
+    # CLI flag: -blocks-storage.swift.application-credential-secret
+    [application_credential_secret: <string> | default = ""]
+
     # OpenStack Swift username.
     # CLI flag: -blocks-storage.swift.username
     [username: <string> | default = ""]

docs/configuration/config-file-reference.md

@@ -563,6 +563,10 @@ s3:
   # CLI flag: -alertmanager-storage.s3.send-content-md5
   [send_content_md5: <boolean> | default = true]
 
+  # The list api version. Supported values are: v1, v2, and ''.
+  # CLI flag: -alertmanager-storage.s3.list-objects-version
+  [list_objects_version: <string> | default = ""]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: alertmanager-storage
   [sse: <s3_sse_config>]
@@ -702,6 +706,18 @@ swift:
   # CLI flag: -alertmanager-storage.swift.auth-url
   [auth_url: <string> | default = ""]
 
+  # OpenStack Swift application credential ID.
+  # CLI flag: -alertmanager-storage.swift.application-credential-id
+  [application_credential_id: <string> | default = ""]
+
+  # OpenStack Swift application credential name.
+  # CLI flag: -alertmanager-storage.swift.application-credential-name
+  [application_credential_name: <string> | default = ""]
+
+  # OpenStack Swift application credential secret.
+  # CLI flag: -alertmanager-storage.swift.application-credential-secret
+  [application_credential_secret: <string> | default = ""]
+
   # OpenStack Swift username.
   # CLI flag: -alertmanager-storage.swift.username
   [username: <string> | default = ""]
@@ -842,6 +858,10 @@ s3:
   # CLI flag: -blocks-storage.s3.send-content-md5
   [send_content_md5: <boolean> | default = true]
 
+  # The list api version. Supported values are: v1, v2, and ''.
+  # CLI flag: -blocks-storage.s3.list-objects-version
+  [list_objects_version: <string> | default = ""]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: blocks-storage
   [sse: <s3_sse_config>]
@@ -981,6 +1001,18 @@ swift:
   # CLI flag: -blocks-storage.swift.auth-url
   [auth_url: <string> | default = ""]
 
+  # OpenStack Swift application credential ID.
+  # CLI flag: -blocks-storage.swift.application-credential-id
+  [application_credential_id: <string> | default = ""]
+
+  # OpenStack Swift application credential name.
+  # CLI flag: -blocks-storage.swift.application-credential-name
+  [application_credential_name: <string> | default = ""]
+
+  # OpenStack Swift application credential secret.
+  # CLI flag: -blocks-storage.swift.application-credential-secret
+  [application_credential_secret: <string> | default = ""]
+
   # OpenStack Swift username.
   # CLI flag: -blocks-storage.swift.username
   [username: <string> | default = ""]
@@ -3041,6 +3073,13 @@ instance_limits:
 # Experimental: Enable string interning for metrics labels.
 # CLI flag: -ingester.labels-string-interning-enabled
 [labels_string_interning_enabled: <boolean> | default = false]
+
+# Disable trimming of matching series chunks based on query Start and End time.
+# When disabled, the result may contain samples outside the queried time range
+# but select performances may be improved. Note that certain query results might
+# change by changing this option.
+# CLI flag: -ingester.disable-chunk-trimming
+[disable_chunk_trimming: <boolean> | default = false]
 ```
 
 ### `ingester_client_config`
@@ -4611,6 +4650,10 @@ s3:
   # CLI flag: -ruler-storage.s3.send-content-md5
   [send_content_md5: <boolean> | default = true]
 
+  # The list api version. Supported values are: v1, v2, and ''.
+  # CLI flag: -ruler-storage.s3.list-objects-version
+  [list_objects_version: <string> | default = ""]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: ruler-storage
   [sse: <s3_sse_config>]
@@ -4750,6 +4793,18 @@ swift:
   # CLI flag: -ruler-storage.swift.auth-url
   [auth_url: <string> | default = ""]
 
+  # OpenStack Swift application credential ID.
+  # CLI flag: -ruler-storage.swift.application-credential-id
+  [application_credential_id: <string> | default = ""]
+
+  # OpenStack Swift application credential name.
+  # CLI flag: -ruler-storage.swift.application-credential-name
+  [application_credential_name: <string> | default = ""]
+
+  # OpenStack Swift application credential secret.
+  # CLI flag: -ruler-storage.swift.application-credential-secret
+  [application_credential_secret: <string> | default = ""]
+
   # OpenStack Swift username.
   # CLI flag: -ruler-storage.swift.username
   [username: <string> | default = ""]
@@ -4898,6 +4953,10 @@ s3:
   # CLI flag: -runtime-config.s3.send-content-md5
   [send_content_md5: <boolean> | default = true]
 
+  # The list api version. Supported values are: v1, v2, and ''.
+  # CLI flag: -runtime-config.s3.list-objects-version
+  [list_objects_version: <string> | default = ""]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: runtime-config
   [sse: <s3_sse_config>]
@@ -5037,6 +5096,18 @@ swift:
   # CLI flag: -runtime-config.swift.auth-url
   [auth_url: <string> | default = ""]
 
+  # OpenStack Swift application credential ID.
+  # CLI flag: -runtime-config.swift.application-credential-id
+  [application_credential_id: <string> | default = ""]
+
+  # OpenStack Swift application credential name.
+  # CLI flag: -runtime-config.swift.application-credential-name
+  [application_credential_name: <string> | default = ""]
+
+  # OpenStack Swift application credential secret.
+  # CLI flag: -runtime-config.swift.application-credential-secret
+  [application_credential_secret: <string> | default = ""]
+
   # OpenStack Swift username.
   # CLI flag: -runtime-config.swift.username
   [username: <string> | default = ""]

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/aws/aws-sdk-go v1.55.5
 	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
 	github.com/cespare/xxhash v1.1.0
-	github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e
+	github.com/cortexproject/promqlsmith v0.0.0-20241102030034-4051538fd914
 	github.com/dustin/go-humanize v1.0.1
 	github.com/efficientgo/core v1.0.0-rc.3
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
@@ -46,14 +46,14 @@ require (
 	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/common v0.60.1
 	// Prometheus maps version 2.x.y to tags v0.x.y.
-	github.com/prometheus/prometheus v0.55.0
+	github.com/prometheus/prometheus v0.55.1-0.20241102120812-a6fd22b9d2c8
 	github.com/segmentio/fasthash v1.0.3
 	github.com/sony/gobreaker v1.0.0
 	github.com/spf13/afero v1.11.0
 	github.com/stretchr/testify v1.9.0
-	github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069
+	github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90
 	github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31
-	github.com/thanos-io/thanos v0.35.2-0.20241011111532-af0900bfd290
+	github.com/thanos-io/thanos v0.35.2-0.20241029125830-62038110b1bc
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20230728070032-dd9e68f319d5
 	go.etcd.io/etcd/api/v3 v3.5.16

go.sum

@@ -944,8 +944,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e h1:nOWmgQD3L/Z0bmm29iDxB7nlqjMnh7yD/PNOx9rnZmA=
-github.com/cortexproject/promqlsmith v0.0.0-20240506042652-6cfdd9739a5e/go.mod h1:+bSqRETXJ1uk2S93m//htzTVqu8DJPvlGEb3bSE9PzI=
+github.com/cortexproject/promqlsmith v0.0.0-20241102030034-4051538fd914 h1:UhI6yOSqMz3ln8FGaZRLbJTKzPHRaVwewoTa6N5PU5k=
+github.com/cortexproject/promqlsmith v0.0.0-20241102030034-4051538fd914/go.mod h1:ypUb6BfnDVr7QrBgAxtzRqZ573swvka0BdCkPqa2A5g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -1600,8 +1600,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/prometheus/prometheus v0.55.0 h1:ITinOi1zr3HemoVWHf679PfRRmpxZOcR4nEvsze6eB0=
-github.com/prometheus/prometheus v0.55.0/go.mod h1:GGS7QlWKCqCbcEzWsVahYIfQwiGhcExkarHyLJTsv6I=
+github.com/prometheus/prometheus v0.55.1-0.20241102120812-a6fd22b9d2c8 h1:hCxAh6+hxwy7dqUPE5ndnilMeCWrqQkJVjPDXtiYRVo=
+github.com/prometheus/prometheus v0.55.1-0.20241102120812-a6fd22b9d2c8/go.mod h1:GGS7QlWKCqCbcEzWsVahYIfQwiGhcExkarHyLJTsv6I=
 github.com/redis/rueidis v1.0.45-alpha.1 h1:69Bu1l7gVC/qDYuGGwMwGg2rjOjSyxESol/Zila62gY=
 github.com/redis/rueidis v1.0.45-alpha.1/go.mod h1:q7BfhDaPt7xxwy2nv2RqQO12/mmHflDjebpcNwWFjms=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -1684,12 +1684,12 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4
 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng=
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
-github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM=
-github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg=
+github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90 h1:+gDIM0kLg4mpwU4RjvG09KiZsPrCy9EBHf8J/uJ+Ve0=
+github.com/thanos-io/objstore v0.0.0-20241010161353-f90c89a0ef90/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4=
 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ=
 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00=
-github.com/thanos-io/thanos v0.35.2-0.20241011111532-af0900bfd290 h1:d58OLbcIC6F3TviRFK85Ucdxbs/3cPI1fcLRBWiNThA=
-github.com/thanos-io/thanos v0.35.2-0.20241011111532-af0900bfd290/go.mod h1:kqF1rQspIAL+rktXL3OSKfiDjJmsCRezQblsR56degY=
+github.com/thanos-io/thanos v0.35.2-0.20241029125830-62038110b1bc h1:AoIDFFb3xjED+mmf5g6bnULDgNJWeXtKyjAQ1CKtqfk=
+github.com/thanos-io/thanos v0.35.2-0.20241029125830-62038110b1bc/go.mod h1:6Q6xfe4mUYCYbrixod2dNZqKCK4J9pGRKqF29/cUr/A=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber/jaeger-client-go v2.28.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=

integration/alertmanager_test.go

@@ -156,7 +156,7 @@ func TestAlertmanagerClustering(t *testing.T) {
 		Bucket:    alertsBucketName,
 		AccessKey: e2edb.MinioAccessKey,
 		SecretKey: e2edb.MinioSecretKey,
-	}, "alertmanager-test")
+	}, "alertmanager-test", nil)
 	require.NoError(t, err)
 
 	// Create and upload an Alertmanager configuration.
@@ -219,7 +219,7 @@ func TestAlertmanagerSharding(t *testing.T) {
 				Bucket:    alertsBucketName,
 				AccessKey: e2edb.MinioAccessKey,
 				SecretKey: e2edb.MinioSecretKey,
-			}, "alertmanager-test")
+			}, "alertmanager-test", nil)
 			require.NoError(t, err)
 
 			// Create and upload Alertmanager configurations.
@@ -537,7 +537,7 @@ func TestAlertmanagerShardingScaling(t *testing.T) {
 				Bucket:    alertsBucketName,
 				AccessKey: e2edb.MinioAccessKey,
 				SecretKey: e2edb.MinioSecretKey,
-			}, "alertmanager-test")
+			}, "alertmanager-test", nil)
 			require.NoError(t, err)
 
 			// Create and upload Alertmanager configurations.

integration/e2e/db/db.go

@@ -69,14 +69,17 @@ func NewETCD() *e2e.HTTPService {
 		9000, // Metrics
 	)
 }
-func NewPrometheus(flags map[string]string) *e2e.HTTPService {
-	return NewPrometheusWithName("prometheus", flags)
+func NewPrometheus(image string, flags map[string]string) *e2e.HTTPService {
+	return NewPrometheusWithName("prometheus", image, flags)
 }
 
-func NewPrometheusWithName(name string, flags map[string]string) *e2e.HTTPService {
+func NewPrometheusWithName(name string, image string, flags map[string]string) *e2e.HTTPService {
+	if image == "" {
+		image = images.Prometheus
+	}
 	prom := e2e.NewHTTPService(
 		name,
-		images.Prometheus,
+		image,
 		e2e.NewCommandWithoutEntrypoint("prometheus", e2e.BuildArgs(e2e.MergeFlags(map[string]string{
 			"--storage.tsdb.path":               filepath.Join(e2e.ContainerSharedDir, "data"),
 			"--storage.tsdb.max-block-duration": "2h",

integration/e2e/scenario_test.go

@@ -57,7 +57,7 @@ func testMinioWorking(t *testing.T, m *e2e.HTTPService) {
 	})
 	require.NoError(t, err)
 
-	bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test")
+	bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test", nil)
 	require.NoError(t, err)
 
 	require.NoError(t, bkt.Upload(ctx, "recipe", bytes.NewReader([]byte("Just go to Pastry Shop and buy."))))
@@ -116,7 +116,7 @@ func TestScenario(t *testing.T) {
 		SecretKey: e2edb.MinioSecretKey,
 	})
 	require.NoError(t, err)
-	bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test")
+	bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test", nil)
 	require.NoError(t, err)
 
 	_, err = bkt.Get(context.Background(), "recipe")
@@ -140,7 +140,7 @@ func TestScenario(t *testing.T) {
 		SecretKey: e2edb.MinioSecretKey,
 	})
 	require.NoError(t, err)
-	bkt, err = s3.NewBucket(log.NewNopLogger(), b, "test")
+	bkt, err = s3.NewBucket(log.NewNopLogger(), b, "test", nil)
 	require.NoError(t, err)
 
 	_, err = bkt.Get(context.Background(), "recipe")

integration/query_frontend_test.go

@@ -641,7 +641,7 @@ func TestQueryFrontendQueryRejection(t *testing.T) {
 		Bucket:    bucketName,
 		AccessKey: e2edb.MinioAccessKey,
 		SecretKey: e2edb.MinioSecretKey,
-	}, "runtime-config-test")
+	}, "runtime-config-test", nil)
 
 	require.NoError(t, err)