Fix: ensure Redshift's _fetch_native_df respects case-sensitivity

sqlmesh/core/engine_adapter/redshift.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import typing as t
 
 import pandas as pd
@@ -26,6 +27,8 @@
     from sqlmesh.core._typing import SchemaName, TableName
     from sqlmesh.core.engine_adapter.base import QueryOrDF
 
+logger = logging.getLogger(__name__)
+
 
 @set_catalog()
 class RedshiftEngineAdapter(
@@ -128,7 +131,23 @@ def _fetch_native_df(
     ) -> pd.DataFrame:
         """Fetches a Pandas DataFrame from the cursor"""
         self.execute(query, quote_identifiers=quote_identifiers)
-        return self.cursor.fetch_dataframe()
+
+        # We manually build the `DataFrame` here because the driver's `fetch_dataframe`
+        # method does not respect the active case-sensitivity configuration.
+        #
+        # Context: https://github.com/aws/amazon-redshift-python-driver/issues/238
+        fetcheddata = self.cursor.fetchall()
+
+        try:
+            columns = [column[0] for column in self.cursor.description]
+        except Exception:
+            columns = None
+            logging.warning(
+                "No row description was found, pandas dataframe will be missing column labels."
+            )
+
+        result = [tuple(row) for row in fetcheddata]
+        return pd.DataFrame(result, columns=columns)
 
     def _create_table_from_source_queries(
         self,

tests/core/engine_adapter/integration/test_integration_redshift.py

@@ -85,3 +85,12 @@ def test_columns(ctx: TestContext):
             {k: columns[k] for k in max_cols}
         ).values()
     ] == ["CHAR(max)", "CHAR(max)", "CHAR(max)", "VARCHAR(max)", "VARCHAR(max)", "VARCHAR(max)"]
+
+
+def test_fetch_native_df_respects_case_sensitivity(ctx: TestContext):
+    adapter = ctx.engine_adapter
+    adapter.execute("SET enable_case_sensitive_identifier TO true")
+    assert adapter.fetchdf('WITH t AS (SELECT 1 AS "C", 2 AS "c") SELECT * FROM t').to_dict() == {
+        "C": {0: 1},
+        "c": {0: 2},
+    }