Merge branch 'main' into Behavior_change_for_mf_timespine_without_yam…

…l_configuration

.changes/unreleased/Features-20240806-144859.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add `order_by` and `limit` fields to saved queries.
+time: 2024-08-06T14:48:59.035914-07:00
+custom:
+  Author: plypaul
+  Issue: "10531"

.changes/unreleased/Features-20241001-134051.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable use of multi-column unique key in snapshots
+time: 2024-10-01T13:40:51.297529-04:00
+custom:
+  Author: gshank
+  Issue: "9992"

.changes/unreleased/Features-20241007-115853.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Ensure microbatch models use same `current_time` value
+time: 2024-10-07T11:58:53.460941-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10819"

.changes/unreleased/Fixes-20241014-212135.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Allow instances of generic data tests to be documented
+time: 2024-10-14T21:21:35.767115+01:00
+custom:
+    Author: aranke
+    Issue: "2578"

.changes/unreleased/Fixes-20241015-121825.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix warnings for models referring to a deprecated model
+time: 2024-10-15T12:18:25.14525-04:00
+custom:
+  Author: gshank
+  Issue: "10833"

.changes/unreleased/Fixes-20241018-135810.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Exclude hook result from results in on-run-end context
+time: 2024-10-18T13:58:10.396884-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "7387"

.changes/unreleased/Under the Hood-20241016-144056.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Remove support and testing for Python 3.8, which is now EOL.
+time: 2024-10-16T14:40:56.451972-04:00
+custom:
+  Author: gshank peterallenwebb
+  Issue: "10861"

.github/workflows/main.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
 
     env:
       TOXENV: "unit"
@@ -139,7 +139,7 @@ jobs:
       - name: generate include
         id: generate-include
         run: |
-          INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' )
+          INCLUDE=('"python-version":"3.9","os":"windows-latest"' '"python-version":"3.9","os":"macos-12"' )
           INCLUDE_GROUPS="["
           for include in ${INCLUDE[@]}; do
               for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
@@ -161,7 +161,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
         os: [ubuntu-20.04]
         split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
         include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
@@ -263,7 +263,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |

.github/workflows/model_performance.yml

@@ -150,7 +150,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install dbt
         run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }}

.github/workflows/schema-check.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Checkout dbt repo
         uses: actions/checkout@v4

.github/workflows/structured-logging-schema-check.yml

@@ -76,7 +76,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install python dependencies
         run: |

.github/workflows/test-repeater.yml

@@ -27,7 +27,6 @@ on:
         description: 'Version of Python to Test Against'
         type: choice
         options:
-          - '3.8'
           - '3.9'
           - '3.10'
           - '3.11'

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 
 exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py|core/dbt/events/core_types_pb2.py|core/dbt/adapters/events/adapter_types_pb2.py)
 
-# Force all unspecified python hooks to run python 3.8
+# Force all unspecified python hooks to run python 3.9
 default_language_version:
   python: python3
 

Dockerfile.test

@@ -33,9 +33,6 @@ RUN apt-get update \
     python-is-python3 \
     python-dev-is-python3 \
     python3-pip \
-    python3.8 \
-    python3.8-dev \
-    python3.8-venv \
     python3.9 \
     python3.9-dev \
     python3.9-venv \

core/dbt/artifacts/resources/v1/saved_query.py

@@ -44,6 +44,8 @@ class QueryParams(dbtClassMixin):
     metrics: List[str]
     group_by: List[str]
     where: Optional[WhereFilterIntersection]
+    order_by: List[str] = field(default_factory=list)
+    limit: Optional[int] = None
 
 
 @dataclass

core/dbt/artifacts/resources/v1/snapshot.py

@@ -19,7 +19,7 @@ class SnapshotMetaColumnNames(dbtClassMixin):
 class SnapshotConfig(NodeConfig):
     materialized: str = "snapshot"
     strategy: Optional[str] = None
-    unique_key: Optional[str] = None
+    unique_key: Optional[Union[str, List[str]]] = None
     target_schema: Optional[str] = None
     target_database: Optional[str] = None
     updated_at: Optional[str] = None

core/dbt/artifacts/schemas/results.py

@@ -10,6 +10,12 @@
 
 @dataclass
 class TimingInfo(dbtClassMixin):
+    """
+    Represents a step in the execution of a node.
+    `name` should be one of: compile, execute, or other
+    Do not call directly, use `collect_timing_info` instead.
+    """
+
     name: str
     started_at: Optional[datetime] = None
     completed_at: Optional[datetime] = None
@@ -21,7 +27,7 @@ def end(self):
         self.completed_at = datetime.utcnow()
 
     def to_msg_dict(self):
-        msg_dict = {"name": self.name}
+        msg_dict = {"name": str(self.name)}
         if self.started_at:
             msg_dict["started_at"] = datetime_to_json_string(self.started_at)
         if self.completed_at:

core/dbt/config/runtime.py

@@ -1,7 +1,8 @@
 import itertools
 import os
 from copy import deepcopy
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from datetime import datetime
 from pathlib import Path
 from typing import (
     Any,
@@ -15,6 +16,8 @@
     Type,
 )
 
+import pytz
+
 from dbt import tracking
 from dbt.adapters.contracts.connection import (
     AdapterRequiredConfig,
@@ -98,6 +101,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
     profile_name: str
     cli_vars: Dict[str, Any]
     dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None
+    invoked_at: datetime = field(default_factory=lambda: datetime.now(pytz.UTC))
 
     def __post_init__(self):
         self.validate()

core/dbt/contracts/graph/manifest.py

@@ -1315,25 +1315,6 @@ def singular_test_lookup(self) -> SingularTestLookup:
     def external_node_unique_ids(self):
         return [node.unique_id for node in self.nodes.values() if node.is_external_node]
 
-    def resolve_refs(
-        self,
-        source_node: ModelNode,
-        current_project: str,  # TODO: ModelNode is overly restrictive typing
-    ) -> List[MaybeNonSource]:
-        resolved_refs: List[MaybeNonSource] = []
-        for ref in source_node.refs:
-            resolved = self.resolve_ref(
-                source_node,
-                ref.name,
-                ref.package,
-                ref.version,
-                current_project,
-                source_node.package_name,
-            )
-            resolved_refs.append(resolved)
-
-        return resolved_refs
-
     # Called by dbt.parser.manifest._process_refs & ManifestLoader.check_for_model_deprecations
     def resolve_ref(
         self,

core/dbt/contracts/graph/unparsed.py

@@ -720,6 +720,8 @@ class UnparsedQueryParams(dbtClassMixin):
     group_by: List[str] = field(default_factory=list)
     # Note: `Union` must be the outermost part of the type annotation for serialization to work properly.
     where: Union[str, List[str], None] = None
+    order_by: List[str] = field(default_factory=list)
+    limit: Optional[int] = None
 
 
 @dataclass

core/dbt/materializations/incremental/microbatch.py

@@ -18,6 +18,7 @@ def __init__(
         is_incremental: bool,
         event_time_start: Optional[datetime],
         event_time_end: Optional[datetime],
+        default_end_time: Optional[datetime] = None,
     ):
         if model.config.incremental_strategy != "microbatch":
             raise DbtInternalError(
@@ -35,10 +36,11 @@ def __init__(
             event_time_start.replace(tzinfo=pytz.UTC) if event_time_start else None
         )
         self.event_time_end = event_time_end.replace(tzinfo=pytz.UTC) if event_time_end else None
+        self.default_end_time = default_end_time or datetime.now(pytz.UTC)
 
     def build_end_time(self):
         """Defaults the end_time to the current time in UTC unless a non `None` event_time_end was provided"""
-        return self.event_time_end or datetime.now(tz=pytz.utc)
+        return self.event_time_end or self.default_end_time
 
     def build_start_time(self, checkpoint: Optional[datetime]):
         """Create a start time based off the passed in checkpoint.

core/dbt/parser/generic_test_builders.py

@@ -136,6 +136,17 @@ def __init__(
         if self.namespace is not None:
             self.package_name = self.namespace
 
+        # If the user has provided a description for this generic test, use it
+        # Then delete the "description" argument to:
+        # 1. Avoid passing it into the test macro
+        # 2. Avoid passing it into the test name synthesis
+        # Otherwise, use an empty string
+        self.description: str = ""
+
+        if "description" in self.args:
+            self.description = self.args["description"]
+            del self.args["description"]
+
         # If the user has provided a custom name for this generic test, use it
         # Then delete the "name" argument to avoid passing it into the test macro
         # Otherwise, use an auto-generated name synthesized from test inputs

core/dbt/parser/manifest.py

@@ -573,36 +573,41 @@ def safe_update_project_parser_files_partially(self, project_parser_files: Dict)
         return project_parser_files
 
     def check_for_model_deprecations(self):
+        # build parent and child_maps
+        self.manifest.build_parent_and_child_maps()
         for node in self.manifest.nodes.values():
-            if isinstance(node, ModelNode) and node.is_past_deprecation_date:
-                warn_or_error(
-                    DeprecatedModel(
-                        model_name=node.name,
-                        model_version=version_to_str(node.version),
-                        deprecation_date=node.deprecation_date.isoformat(),
+            if isinstance(node, ModelNode) and node.deprecation_date:
+                if node.is_past_deprecation_date:
+                    warn_or_error(
+                        DeprecatedModel(
+                            model_name=node.name,
+                            model_version=version_to_str(node.version),
+                            deprecation_date=node.deprecation_date.isoformat(),
+                        )
                     )
-                )
-
-                resolved_refs = self.manifest.resolve_refs(node, self.root_project.project_name)
-                resolved_model_refs = [r for r in resolved_refs if isinstance(r, ModelNode)]
-                node.depends_on
-                for resolved_ref in resolved_model_refs:
-                    if resolved_ref.deprecation_date:
-                        if resolved_ref.is_past_deprecation_date:
-                            event_cls = DeprecatedReference
-                        else:
-                            event_cls = UpcomingReferenceDeprecation
-
-                        warn_or_error(
-                            event_cls(
-                                model_name=node.name,
-                                ref_model_package=resolved_ref.package_name,
-                                ref_model_name=resolved_ref.name,
-                                ref_model_version=version_to_str(resolved_ref.version),
-                                ref_model_latest_version=str(resolved_ref.latest_version),
-                                ref_model_deprecation_date=resolved_ref.deprecation_date.isoformat(),
-                            )
+                # At this point _process_refs should already have been called, and
+                # we just rebuilt the parent and child maps.
+                # Get the child_nodes and check for deprecations.
+                child_nodes = self.manifest.child_map[node.unique_id]
+                for child_unique_id in child_nodes:
+                    child_node = self.manifest.nodes[child_unique_id]
+                    if not isinstance(child_node, ModelNode):
+                        continue
+                    if node.is_past_deprecation_date:
+                        event_cls = DeprecatedReference
+                    else:
+                        event_cls = UpcomingReferenceDeprecation
+
+                    warn_or_error(
+                        event_cls(
+                            model_name=child_node.name,
+                            ref_model_package=node.package_name,
+                            ref_model_name=node.name,
+                            ref_model_version=version_to_str(node.version),
+                            ref_model_latest_version=str(node.latest_version),
+                            ref_model_deprecation_date=node.deprecation_date.isoformat(),
                         )
+                    )
 
     def check_for_spaces_in_resource_names(self):
         """Validates that resource names do not contain spaces

core/dbt/parser/schema_generic_tests.py

@@ -96,6 +96,7 @@ def create_test_node(
         test_metadata: Dict[str, Any],
         file_key_name: str,
         column_name: Optional[str],
+        description: str,
     ) -> GenericTestNode:
 
         HASH_LENGTH = 10
@@ -134,6 +135,7 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List
             "column_name": column_name,
             "checksum": FileHash.empty().to_dict(omit_none=True),
             "file_key_name": file_key_name,
+            "description": description,
         }
         try:
             GenericTestNode.validate(dct)
@@ -229,6 +231,7 @@ def parse_generic_test(
             column_name=column_name,
             test_metadata=metadata,
             file_key_name=file_key_name,
+            description=builder.description,
         )
         self.render_test_update(node, config, builder, schema_file_id)
 

core/dbt/parser/schema_yaml_readers.py

@@ -788,6 +788,8 @@ def _get_query_params(self, unparsed: UnparsedQueryParams) -> QueryParams:
             group_by=unparsed.group_by,
             metrics=unparsed.metrics,
             where=parse_where_filter(unparsed.where),
+            order_by=unparsed.order_by,
+            limit=unparsed.limit,
         )
 
     def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: