Merge branch 'main' into persistent-storage-2

QuesmaOrg · Nov 11, 2024 · 532f39e · 532f39e
2 parents 576b195 + 5a41028
commit 532f39e
Show file tree

Hide file tree

Showing 47 changed files with 1,936 additions and 846 deletions.
diff --git a/NOTICE.MD b/NOTICE.MD
@@ -853,6 +853,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+#### Module  : github.com/gorilla/securecookie
+Version : v1.1.2
+Time    : 2023-10-18T11:18:40Z
+Licence : BSD-3-Clause
+
+Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
+
+Copyright (c) 2023 The Gorilla Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+	 * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+	 * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+	 * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 #### Module  : github.com/gorilla/sessions
 Version : v1.4.0
@@ -1660,37 +1697,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-#### Module  : github.com/knadh/koanf/providers/env
-Version : v1.0.0
-Time    : 2024-09-26T07:31:31Z
-Licence : MIT
-
-Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/providers/[email protected]/LICENSE:
-
-The MIT License
-
-Copyright (c) 2019, Kailash Nadh. https://github.com/knadh
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
-
 --------------------------------------------------------------------------------
 #### Module  : github.com/knadh/koanf/providers/file
 Version : v1.1.2
@@ -1724,11 +1730,11 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------------------
 #### Module  : github.com/knadh/koanf/v2
-Version : v2.1.1
-Time    : 2024-04-03T07:16:15Z
+Version : v2.1.2
+Time    : 2024-11-06T08:48:20Z
 Licence : MIT
 
-Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/[email protected].1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/knadh/koanf/[email protected].2/LICENSE:
 
 The MIT License
 
@@ -3864,6 +3870,39 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+--------------------------------------------------------------------------------
+#### Module  : github.com/creack/pty
+Version : v1.1.9
+Time    : 2019-09-25T15:36:33Z
+Licence : MIT
+
+Contents of probable licence file $GOMODCACHE/github.com/creack/[email protected]/LICENSE:
+
+Copyright (c) 2011 Keith Rarick
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall
+be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 #### Module  : github.com/davecgh/go-spew
 Version : v1.1.2-0.20180830191138-d8f796af33cc
@@ -5595,11 +5634,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 #### Module  : github.com/go-viper/mapstructure/v2
-Version : v2.0.0-alpha.1
-Time    : 2023-12-19T21:02:02Z
+Version : v2.2.1
+Time    : 2024-09-22T22:42:12Z
 Licence : MIT
 
-Contents of probable licence file $GOMODCACHE/github.com/go-viper/mapstructure/v2@v2.0.0-alpha.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/go-viper/mapstructure/v2@v2.2.1/LICENSE:
 
 The MIT License (MIT)
 
@@ -6152,43 +6191,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
---------------------------------------------------------------------------------
-#### Module  : github.com/gorilla/securecookie
-Version : v1.1.2
-Time    : 2023-10-18T11:18:40Z
-Licence : BSD-3-Clause
-
-Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
-
-Copyright (c) 2023 The Gorilla Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-	 * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-	 * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-	 * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 #### Module  : github.com/gorilla/websocket
 Version : v1.4.2

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@
 <a href="https://www.linkedin.com/company/quesma">LinkedIn</a> | 
 <a href="https://twitter.com/QuesmaOrg">Twitter</a> | 
 <a href="https://www.youtube.com/@QuesmaOrg">YouTube</a> | 
-<a href="https://eap.quesma.com/eap-docs">Docs</a>
+<a href="https://docs.quesma.com/eap-docs">Docs</a>
 </p>
 
 Quesma is an actively developed database gateway currently in pre-alpha Early Access. Route your queries through a translation layer that seamlessly connects your apps with the modern database platforms of your choice. No more query refactoring during migrations.

diff --git a/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template b/ci/it/configs/quesma-with-dual-writes-and-common-table.yml.template
@@ -33,11 +33,13 @@ processors:
         logs-dual-query:
           target: [ c, e ]
         logs-4:
-          useCommonTable: true
-          target: [ c ]
+          target:
+            - c:
+                useCommonTable: true
         logs-5:
-          useCommonTable: true
-          target: [ c ]
+          target:
+            - c:
+                useCommonTable: true
         "*":
           target: [ e ]
   - name: IP
@@ -53,13 +55,13 @@ processors:
         logs-dual-query:
           target: [ c, e ]
         logs-4:
-          useCommonTable: true
-          target: [ c ]
+          target:
+            - c:
+                useCommonTable: true
+        logs-5:
+          target:
         "*":
           target: [ e ]
-        logs-5:
-          useCommonTable: true
-          target: [  ]
 
 pipelines:
   - name: my-elasticsearch-proxy-read

diff --git a/docs/public/README.md b/docs/public/README.md
@@ -1,6 +1,6 @@
-# Quesma EAP documentation
+# Quesma documentation
 
-This folder contains our EAP documentation available at https://eap.quesma.com.
+This folder contains our documentation available at https://docs.quesma.com (formerly https://eap.quesma.com).
 These docs are just static files generated with [Vitepress](https://vitepress.dev) and published via CloudFlare Pages.
 
 

diff --git a/docs/public/docs/.vitepress/config.mts b/docs/public/docs/.vitepress/config.mts
@@ -41,8 +41,6 @@ export default defineConfig({
                   { text: 'Query Hydrolix tables as Elasticsearch indices', link: '/example-2-0'}
                 ]
               },
-              //{ text: 'Scenario I', link: '/scenario-1' },
-              //{ text: 'Reference Docker compose configurations', link: '/reference-conf' }
             ],
           },
           { text: 'Advanced configuration',

diff --git a/docs/public/docs/config-primer.md b/docs/public/docs/config-primer.md
@@ -70,12 +70,17 @@ frontendConnectors:
       listenPort: 8080
 ```
 
+The supported configuration options for frontend connectors (under `config`):
+* `listenPort` - port number on which the frontend connector will listen for incoming requests
+* `disableAuth` - when set to `true`, disables authentication for incoming requests (optional, defaults to false). If you use Elasticsearch/Kibana without authentication, set it to `true`.
+
+
 #### Backend connectors
 
 Backend connector has to have a `name`, `type` and `config` fields.
 * `name` is a unique identifier for the connector
 * `type` specifies the type of the connector.\
-  At this moment, only three backend connector types are allowed: `elasticsearch`, `clickhouse` (used for ClickHouse Cloud SaaS service, `clickhouse-os` and `hydrolix`.
+  At this moment, only three backend connector types are allowed: `elasticsearch`, `clickhouse` (used for ClickHouse Cloud SaaS service), `clickhouse-os` (self-hosted ClickHouse) and `hydrolix`.
 * `config` is a set of configuration options for the connector.
 ```yaml
 backendConnectors:
@@ -84,16 +89,25 @@ backendConnectors:
     config:
       user: "elastic"
       password: "change-me"
-      url: "http://elasticsearch:9200"
+      url: "http://192.168.0.7:9200"
   - name: my-clickhouse-data-source
     type: clickhouse-os
     config:
       user: "username"
       password: "username-is-password"
       database: "dbname"
-      url: "clickhouse://clickhouse:9000"
+      url: "clickhouse://192.168.0.9:9000"
 ```
-**WARNING:** When connecting to ClickHouse or Hydrolix, only the native protocol connection (`clickhouse://`) is supported.
+
+The supported configuration options for backend connectors (under `config`):
+* `url` - connection string to the backend service in a URL format (`protocol://host:port`):
+  * for Elastic/OpenSearch the expected format is `http://host:port` (Elastic/OpenSearch default port is 9200)
+  * for ClickHouse/Hydrolix the expected format is `clickhouse://host:port` (ClickHouse default port is 9000, ClickHouse/Hydrolix default encrypted port is 9440). Note that Quesma supports only the ClickHouse native protocol  (`clickhouse://`) and does not support the HTTP protocol.
+* `user` - username for authentication
+* `password` - password for authentication 
+* `database` - name of the database to connect to. It is optional for ClickHouse, but strictly required for Hydrolix, where it is also referred as "project".
+* `adminUrl` - URL for administrative operations to render a handy link in Quesma management UI (optional)
+* `disableTLS` - when set to true, disables TLS for the connection (optional)
 
 ### Processors
 
@@ -178,14 +192,33 @@ The configuration for an index consists of the following configuration options:
    will dual write ingest requests to `my_index` to both ElasticSearch and ClickHouse.
    Note that ElasticSearch/OpenSearch is the only supported backend for the `*` entry.
    If no targets are provided (example: `target: []`) in the configuration of an index in the ingest processor, ingest for that index will be disabled and incoming data will be dropped.
-- `override` (optional): override the name of table in Hydrolix/ClickHouse (by default Quesma uses the same table name as the index name)
-- `useCommonTable` (optional): if enabled, Quesma will store data in a single Hydrolix/ClickHouse table named `quesma_common_table`. See [ingest documentation](/ingest.md) for more details.
+
+   Some backend connectors have additional attributes which may be used. For example the following configuration sets `useCommonTable` for `backend-clickhouse` target:
+   ```yaml
+      my_index:
+        target:
+          - backend-clickhouse:
+              useCommonTable: true
+   ```
+   Currently only the ClickHouse backend connector supports the following attributes:
+     - `useCommonTable` (optional): if enabled, Quesma will store data in a single Hydrolix/ClickHouse table named `quesma_common_table`. See [ingest documentation](/ingest.md) for more details.
+     - `tableName` (optional): override the name of table in Hydrolix/ClickHouse (by default Quesma uses the same table name as the index name)
+- `schemaOverrides` (optional): manual overrides of schema information for an index. Quesma infers schema for an index based on the data ingested and the schema information fetched from ClickHouse. `schemaOverrides` allows you to override this inferred schema with for some fields. For example the following configuration:
+    ```yaml
+      my_index:
+        target: [ backend-clickhouse ]
+        schemaOverrides:
+          "product_name":
+            type: "text"
+    ```
+    changes the type of `product_name` field to `text`. Note: `schemaOverrides` are currently not supported in `*` configuration.
 
 ## Optional configuration options
 
 ### Quesma licensing configuration
 
-In order to be able to use `hydrolix` or `clickhouse` backend connectors, one needs to supply `licenseKey` in the configuration file. Contact us at [email protected] if you need one.
+In order to be able to use `hydrolix` or `clickhouse` backend connectors Quesma requires a commercial license key (supplied in the `licenseKey` field of the configuration file). 
+Contact us at [email protected] if you need one.
 ```yaml
 licenseKey: ZXlKcGJuTjBZV3hz...
 ```  

diff --git a/docs/public/docs/example-1.md b/docs/public/docs/example-1.md
@@ -32,7 +32,7 @@ flowchart LR
       - name: backend-elasticsearch
         type: elasticsearch
         config:
-          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE
+          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE, for example: http://192.168.0.7:9200
           user: #PLACE_YOUR_ELASTICSEARCH_USERNAME_HERE
           password: #PLACE_YOUR_ELASTICSEARCH_PASSWORD_HERE
     processors:

diff --git a/docs/public/docs/example-2-0-clickhouse-specific.md b/docs/public/docs/example-2-0-clickhouse-specific.md
@@ -55,14 +55,14 @@ flowchart LR
       - name: minimal-elasticsearch
         type: elasticsearch
         config:
-          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE
+          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE, for example: http://192.168.0.7:9200
           user: #PLACE_YOUR_ELASTICSEARCH_USERNAME_HERE
           password: #PLACE_YOUR_ELASTICSEARCH_PASSWORD_HERE
       - name: clickhouse-instance
         type: clickhouse-os
         #type: clickhouse # use for ClickHouse cloud service only
         config:
-          url: #PLACE_YOUR_CLICKHOUSE_URL_HERE
+          url: #PLACE_YOUR_CLICKHOUSE_URL_HERE, for example: clickhouse://192.168.0.7:9000
           user: #PLACE_YOUR_CLICKHOUSE_USER_HERE
           password: #PLACE_YOUR_CLICKHOUSE_PASSWORD_HERE
           database: #PLACE_YOUR_CLICKHOUSE_DATABASE_NAME_HERE

diff --git a/docs/public/docs/example-2-0.md b/docs/public/docs/example-2-0.md
@@ -42,13 +42,13 @@ flowchart LR
       - name: minimal-elasticsearch
         type: elasticsearch
         config:
-          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE
+          url: #PLACE_YOUR_ELASTICSEARCH_URL_HERE, for example: http://192.168.0.7:9200
           user: #PLACE_YOUR_ELASTICSEARCH_USERNAME_HERE
           password: #PLACE_YOUR_ELASTICSEARCH_PASSWORD_HERE
       - name: hydrolix-instance
         type: hydrolix
         config:
-          url: #PLACE_YOUR_HYDROLIX_URL_HERE
+          url: #PLACE_YOUR_HYDROLIX_URL_HERE, for example: clickhouse://companyname.hydrolix.live:9440
           user: #PLACE_YOUR_HYDROLIX_USER_HERE
           password: #PLACE_YOUR_HYDROLIX_PASSWORD_HERE
           database: #PLACE_YOUR_HYDROLIX_DATABASE_NAME_HERE