Skip to content

Commit

Permalink
Change Firestore Metric Field Name to Load into Snowflake (#321)
Browse files Browse the repository at this point in the history
### Description
- `response_received_at` is a nullable field. Specifically, if a request
errored out, then this field is null.
- Changing to put `sent_at` field into snowflake's `created_at` column
instead

### Tests
- Tested manually on dev server
<img width="680" alt="image"
src="https://github.com/astronomer/ask-astro/assets/26350341/00d19423-3947-4896-8f62-00ddd343dc55">
Verified in snowflake
<img width="1910" alt="image"
src="https://github.com/astronomer/ask-astro/assets/26350341/82b0199e-cfd0-4607-9642-bbf4dfd0bac5">
  • Loading branch information
davidgxue authored Mar 19, 2024
1 parent bce6232 commit 20ef1c7
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions airflow/dags/metrcis/load_firestore_to_snowflake.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def load_request_data_from_firestore() -> list[tuple[str, int, bool, int]]:
end_ts = time.mktime(yesterday.timetuple())

docs = (
requests_col.where(filter=FieldFilter("response_received_at", ">=", start_ts))
.where(filter=FieldFilter("response_received_at", "<", end_ts))
requests_col.where(filter=FieldFilter("sent_at", ">=", start_ts))
.where(filter=FieldFilter("sent_at", "<", end_ts))
.stream()
)

Expand All @@ -60,9 +60,9 @@ def load_request_data_from_firestore() -> list[tuple[str, int, bool, int]]:
uuid = doc_dict["uuid"]
score = doc_dict.get("score")
status = doc_dict.get("status") == "complete"
response_received_at = datetime.fromtimestamp(doc_dict.get("response_received_at"))
sent_at = datetime.fromtimestamp(doc_dict.get("sent_at"))
client = doc_dict.get("client")
rows.append((uuid, score, status, response_received_at, client))
rows.append((uuid, score, status, sent_at, client))

return rows

Expand Down

0 comments on commit 20ef1c7

Please sign in to comment.