Merge branch 'main' into patch-1

.github/CODEOWNERS

@@ -1 +1 @@
-* @snowflakedb/snowcli
+* @snowflakedb/ORM

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
+        exclude: '\.ambr$'
     -   id: end-of-file-fixer
     -   id: check-yaml
         exclude: .github/repo_meta.yaml

DESCRIPTION.md

@@ -9,11 +9,16 @@ Source code is also available at:
 
 # Release Notes
 
-- (Unreleased)
+- v1.7.0(November 22, 2024)
 
   - Add support for dynamic tables and required options
   - Add support for hybrid tables
   - Fixed SAWarning when registering functions with existing name in default namespace
+  - Update options to be defined in key arguments instead of arguments.
+  - Add support for refresh_mode option in DynamicTable
+  - Add support for iceberg table with Snowflake Catalog
+  - Fix cluster by option to support explicit expressions
+  - Add support for MAP datatype
 
 - v1.6.1(July 9, 2024)
 

README.md

@@ -340,7 +340,7 @@ This example shows how to create a table with two columns, `id` and `name`, as t
 t = Table('myuser', metadata,
     Column('id', Integer, primary_key=True),
     Column('name', String),
-    snowflake_clusterby=['id', 'name'], ...
+    snowflake_clusterby=['id', 'name', text('id > 5')], ...
 )
 metadata.create_all(engine)
 ```

pyproject.toml

@@ -111,7 +111,7 @@ line-length = 88
 line-length = 88
 
 [tool.pytest.ini_options]
-addopts = "-m 'not feature_max_lob_size and not aws'"
+addopts = "-m 'not feature_max_lob_size and not aws and not requires_external_volume'"
 markers = [
   # Optional dependency groups markers
   "lambda: AWS lambda tests",
@@ -128,6 +128,7 @@ markers = [
   # Other markers
   "timeout: tests that need a timeout time",
   "internal: tests that could but should only run on our internal CI",
+  "requires_external_volume: tests that needs a external volume to be executed",
   "external: tests that could but should only run on our external CI",
   "feature_max_lob_size: tests that could but should only run on our external CI",
 ]

src/snowflake/sqlalchemy/__init__.py

@@ -9,7 +9,7 @@
 else:
     import importlib.metadata as importlib_metadata
 
-from sqlalchemy.types import (
+from sqlalchemy.types import (  # noqa
     BIGINT,
     BINARY,
     BOOLEAN,
@@ -27,8 +27,8 @@
     VARCHAR,
 )
 
-from . import base, snowdialect
-from .custom_commands import (
+from . import base, snowdialect  # noqa
+from .custom_commands import (  # noqa
     AWSBucket,
     AzureContainer,
     CopyFormatter,
@@ -41,7 +41,7 @@
     MergeInto,
     PARQUETFormatter,
 )
-from .custom_types import (
+from .custom_types import (  # noqa
     ARRAY,
     BYTEINT,
     CHARACTER,
@@ -50,6 +50,7 @@
     FIXED,
     GEOGRAPHY,
     GEOMETRY,
+    MAP,
     NUMBER,
     OBJECT,
     STRING,
@@ -61,15 +62,30 @@
     VARBINARY,
     VARIANT,
 )
-from .sql.custom_schema import DynamicTable, HybridTable
-from .sql.custom_schema.options import AsQuery, TargetLag, TimeUnit, Warehouse
-from .util import _url as URL
+from .sql.custom_schema import (  # noqa
+    DynamicTable,
+    HybridTable,
+    IcebergTable,
+    SnowflakeTable,
+)
+from .sql.custom_schema.options import (  # noqa
+    AsQueryOption,
+    ClusterByOption,
+    IdentifierOption,
+    KeywordOption,
+    LiteralOption,
+    SnowflakeKeyword,
+    TableOptionKey,
+    TargetLagOption,
+    TimeUnit,
+)
+from .util import _url as URL  # noqa
 
 base.dialect = dialect = snowdialect.dialect
 
 __version__ = importlib_metadata.version("snowflake-sqlalchemy")
 
-__all__ = (
+_custom_types = (
     "BIGINT",
     "BINARY",
     "BOOLEAN",
@@ -104,6 +120,10 @@
     "TINYINT",
     "VARBINARY",
     "VARIANT",
+    "MAP",
+)
+
+_custom_commands = (
     "MergeInto",
     "CSVFormatter",
     "JSONFormatter",
@@ -115,10 +135,28 @@
     "ExternalStage",
     "CreateStage",
     "CreateFileFormat",
-    "DynamicTable",
-    "AsQuery",
-    "TargetLag",
+)
+
+_custom_tables = ("HybridTable", "DynamicTable", "IcebergTable", "SnowflakeTable")
+
+_custom_table_options = (
+    "AsQueryOption",
+    "TargetLagOption",
+    "LiteralOption",
+    "IdentifierOption",
+    "KeywordOption",
+    "ClusterByOption",
+)
+
+_enums = (
     "TimeUnit",
-    "Warehouse",
-    "HybridTable",
+    "TableOptionKey",
+    "SnowflakeKeyword",
+)
+__all__ = (
+    *_custom_types,
+    *_custom_commands,
+    *_custom_tables,
+    *_custom_table_options,
+    *_enums,
 )

src/snowflake/sqlalchemy/_constants.py

@@ -11,3 +11,4 @@
 APPLICATION_NAME = "SnowflakeSQLAlchemy"
 SNOWFLAKE_SQLALCHEMY_VERSION = VERSION
 DIALECT_NAME = "snowflake"
+NOT_NULL = "NOT NULL"

src/snowflake/sqlalchemy/base.py

@@ -5,6 +5,7 @@
 import itertools
 import operator
 import re
+from typing import List
 
 from sqlalchemy import exc as sa_exc
 from sqlalchemy import inspect, sql
@@ -26,8 +27,14 @@
     ExternalStage,
 )
 
+from ._constants import NOT_NULL
+from .exc import (
+    CustomOptionsAreOnlySupportedOnSnowflakeTables,
+    UnexpectedOptionTypeError,
+)
 from .functions import flatten
-from .sql.custom_schema.options.table_option_base import TableOptionBase
+from .sql.custom_schema.custom_table_base import CustomTableBase
+from .sql.custom_schema.options.table_option import TableOption
 from .util import (
     _find_left_clause_to_join_from,
     _set_connection_interpolate_empty_sequences,
@@ -902,15 +909,15 @@ def handle_cluster_by(self, table):
         ...     metadata,
         ...     sa.Column('id', sa.Integer, primary_key=True),
         ...     sa.Column('name', sa.String),
-        ...     snowflake_clusterby=['id', 'name']
+        ...     snowflake_clusterby=['id', 'name', text("id > 5")]
         ... )
         >>> print(CreateTable(user).compile(engine))
         <BLANKLINE>
         CREATE TABLE "user" (
             id INTEGER NOT NULL AUTOINCREMENT,
             name VARCHAR,
             PRIMARY KEY (id)
-        ) CLUSTER BY (id, name)
+        ) CLUSTER BY (id, name, id > 5)
         <BLANKLINE>
         <BLANKLINE>
         """
@@ -919,22 +926,37 @@ def handle_cluster_by(self, table):
         cluster = info.get("clusterby")
         if cluster:
             text += " CLUSTER BY ({})".format(
-                ", ".join(self.denormalize_column_name(key) for key in cluster)
+                ", ".join(
+                    (
+                        self.denormalize_column_name(key)
+                        if isinstance(key, str)
+                        else str(key)
+                    )
+                    for key in cluster
+                )
             )
         return text
 
     def post_create_table(self, table):
         text = self.handle_cluster_by(table)
-        options = [
-            option
-            for _, option in table.dialect_options[DIALECT_NAME].items()
-            if isinstance(option, TableOptionBase)
-        ]
-        options.sort(
-            key=lambda x: (x.__priority__.value, x.__option_name__), reverse=True
-        )
-        for option in options:
-            text += "\t" + option.render_option(self)
+        options = []
+        invalid_options: List[str] = []
+
+        for key, option in table.dialect_options[DIALECT_NAME].items():
+            if isinstance(option, TableOption):
+                options.append(option)
+            elif key not in ["clusterby", "*"]:
+                invalid_options.append(key)
+
+        if len(invalid_options) > 0:
+            raise UnexpectedOptionTypeError(sorted(invalid_options))
+
+        if isinstance(table, CustomTableBase):
+            options.sort(key=lambda x: (x.priority.value, x.option_name), reverse=True)
+            for option in options:
+                text += "\t" + option.render_option(self)
+        elif len(options) > 0:
+            raise CustomOptionsAreOnlySupportedOnSnowflakeTables()
 
         return text
 
@@ -1050,6 +1072,12 @@ def visit_TINYINT(self, type_, **kw):
     def visit_VARIANT(self, type_, **kw):
         return "VARIANT"
 
+    def visit_MAP(self, type_, **kw):
+        not_null = f" {NOT_NULL}" if type_.not_null else ""
+        return (
+            f"MAP({type_.key_type.compile()}, {type_.value_type.compile()}{not_null})"
+        )
+
     def visit_ARRAY(self, type_, **kw):
         return "ARRAY"
 

src/snowflake/sqlalchemy/custom_types.py

@@ -37,6 +37,26 @@ class VARIANT(SnowflakeType):
     __visit_name__ = "VARIANT"
 
 
+class StructuredType(SnowflakeType):
+    def __init__(self):
+        super().__init__()
+
+
+class MAP(StructuredType):
+    __visit_name__ = "MAP"
+
+    def __init__(
+        self,
+        key_type: sqltypes.TypeEngine,
+        value_type: sqltypes.TypeEngine,
+        not_null: bool = False,
+    ):
+        self.key_type = key_type
+        self.value_type = value_type
+        self.not_null = not_null
+        super().__init__()
+
+
 class OBJECT(SnowflakeType):
     __visit_name__ = "OBJECT"
 

src/snowflake/sqlalchemy/exc.py

@@ -0,0 +1,82 @@
+#
+# Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
+
+from typing import List
+
+from sqlalchemy.exc import ArgumentError
+
+
+class NoPrimaryKeyError(ArgumentError):
+    def __init__(self, target: str):
+        super().__init__(f"Table {target} required primary key.")
+
+
+class UnsupportedPrimaryKeysAndForeignKeysError(ArgumentError):
+    def __init__(self, target: str):
+        super().__init__(f"Primary key and foreign keys are not supported in {target}.")
+
+
+class RequiredParametersNotProvidedError(ArgumentError):
+    def __init__(self, target: str, parameters: List[str]):
+        super().__init__(
+            f"{target} requires the following parameters: %s." % ", ".join(parameters)
+        )
+
+
+class UnexpectedTableOptionKeyError(ArgumentError):
+    def __init__(self, expected: str, actual: str):
+        super().__init__(f"Expected table option {expected} but got {actual}.")
+
+
+class OptionKeyNotProvidedError(ArgumentError):
+    def __init__(self, target: str):
+        super().__init__(
+            f"Expected option key in {target} option but got NoneType instead."
+        )
+
+
+class UnexpectedOptionParameterTypeError(ArgumentError):
+    def __init__(self, parameter_name: str, target: str, types: List[str]):
+        super().__init__(
+            f"Parameter {parameter_name} of {target} requires to be one"
+            f" of following types: {', '.join(types)}."
+        )
+
+
+class CustomOptionsAreOnlySupportedOnSnowflakeTables(ArgumentError):
+    def __init__(self):
+        super().__init__(
+            "Identifier, Literal, TargetLag and other custom options are only supported on Snowflake tables."
+        )
+
+
+class UnexpectedOptionTypeError(ArgumentError):
+    def __init__(self, options: List[str]):
+        super().__init__(
+            f"The following options are either unsupported or should be defined using a Snowflake table: {', '.join(options)}."
+        )
+
+
+class InvalidTableParameterTypeError(ArgumentError):
+    def __init__(self, name: str, input_type: str, expected_types: List[str]):
+        expected_types_str = "', '".join(expected_types)
+        super().__init__(
+            f"Invalid parameter type '{input_type}' provided for '{name}'. "
+            f"Expected one of the following types: '{expected_types_str}'.\n"
+        )
+
+
+class MultipleErrors(ArgumentError):
+    def __init__(self, errors):
+        self.errors = errors
+
+    def __str__(self):
+        return "".join(str(e) for e in self.errors)
+
+
+class StructuredTypeNotSupportedInTableColumnsError(ArgumentError):
+    def __init__(self, table_type: str, table_name: str, column_name: str):
+        super().__init__(
+            f"Column '{column_name}' is of a structured type, which is only supported on Iceberg tables. "
+            f"The table '{table_name}' is of type '{table_type}', not Iceberg."
+        )