-
Notifications
You must be signed in to change notification settings - Fork 0
/
prefect.yaml
157 lines (151 loc) · 6.09 KB
/
prefect.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# Welcome to your prefect.yaml file! You can use this file for storing and managing
# configuration for deploying your flows. We recommend committing this file to source
# control along with your flow code.
# Generic metadata about this project
name: unsplash-photo-trends
prefect-version: "{{ prefect.variables.prefect_version }}"
# build section allows you to manage and build docker images
build:
- prefect_docker.deployments.steps.build_docker_image:
id: build-image
requires: prefect-docker>=0.3.1
dockerfile: auto
image_name: "{{ prefect.variables.gcp_default_region }}-docker.pkg.dev/{{ prefect.variables.gcp_project_id }}/prefect-{{ prefect.variables.unsplash_photo_trends_env }}/prefect"
tag: "{{ prefect.variables.prefect_version }}-python{{ prefect.variables.python_version }}"
# push section allows you to manage if and how this project is uploaded to remote locations
push:
- prefect_docker.deployments.steps.push_docker_image:
requires: prefect-docker>=0.3.1
image_name: "{{ build-image.image_name }}"
tag: "{{ build-image.tag }}"
# pull section allows you to provide instructions for cloning this project in remote locations
pull:
- prefect.deployments.steps.utility.run_shell_script:
id: get-git-branch
script: git symbolic-ref --short HEAD
stream_output: true
- prefect.deployments.steps.set_working_directory:
id: set-working-directory
directory: "/opt/prefect/{{ prefect.variables.gcp_project_id }}"
# - prefect.deployments.steps.git_clone:
# id: git-clone
# repository: "{{ prefect.variables.github_repo }}"
# branch: "{{ get-git-branch.stdout }}"
- prefect.deployments.steps.pip_install_requirements:
id: pip-install-requirements
requirements_file: requirements.txt
stream_output: true
# @see https://docs.prefect.io/2.13.7/guides/prefect-deploy/?h=prefect.yaml#reusing-configuration-across-deployments
definitions:
work_pools:
dev-cloud_run_work_pool: &dev_cloud_run_work_pool
name: "dev-cloud-run-push-work-pool"
work_queue_name: null
job_variables:
image: "{{ build-image.image }}"
test-cloud_run_work_pool: &test_cloud_run_work_pool
name: "test-cloud-run-push-work-pool"
work_queue_name: null
job_variables:
image: "{{ build-image.image }}"
prod-cloud_run_work_pool: &prod_cloud_run_work_pool
name: "prod-cloud-run-push-work-pool"
work_queue_name: null
job_variables:
image: "{{ build-image.image }}"
# the deployments section allows you to provide configuration for deploying flows
deployments:
# Test Environment
- name: "healthcheck-test"
entrypoint: src/prefect/healthcheck.py:healthcheck
work_pool: *test_cloud_run_work_pool
schedule: null
- name: "ingest-monthly-platform-stats-gcs-test"
entrypoint: src/prefect/ingest_monthly_platform_stats_gcs.py:ingest_monthly_platform_stats_gcs
work_pool: *test_cloud_run_work_pool
schedule:
cron: "0 8 * * *"
timezone: "Europe/Berlin"
- name: "ingest-topics-gcs-test"
entrypoint: src/prefect/ingest_topics_gcs.py:ingest_topics_gcs
work_pool: *test_cloud_run_work_pool
schedule:
cron: "0 9 * * *"
timezone: "Europe/Berlin"
- name: "ingest-photos-napi-gcs-test"
entrypoint: src/prefect/ingest_photos_napi_gcs.py:ingest_photos_napi_gcs
work_pool: *test_cloud_run_work_pool
schedule:
cron: "*/10 * * * *" # every N min
timezone: "Europe/Berlin"
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
per_page: 30
proxy_type: datacenter
- name: "ingest-photos-expanded-napi-bigquery-test"
entrypoint: src/prefect/ingest_photos_expanded_napi_bigquery.py:ingest_photos_expanded_napi_bigquery
work_pool: *test_cloud_run_work_pool
schedule: null
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
proxy_type: datacenter
batch_size: 30
total_record_size: 60
- name: "ingest-photos-gcs-test"
entrypoint: src/prefect/ingest_photos_gcs.py:ingest_photos_gcs
work_pool: *test_cloud_run_work_pool
schedule: null
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
proxy_type: datacenter
batch_size: 20
total_record_size: 20
# Production Environment
- name: "healthcheck-prod"
entrypoint: src/prefect/healthcheck.py:healthcheck
work_pool: *prod_cloud_run_work_pool
schedule: null
- name: "ingest-monthly-platform-stats-gcs-prod"
entrypoint: src/prefect/ingest_monthly_platform_stats_gcs.py:ingest_monthly_platform_stats_gcs
work_pool: *prod_cloud_run_work_pool
schedule:
cron: "0 8 * * *"
timezone: "Europe/Berlin"
- name: "ingest-topics-gcs-prod"
entrypoint: src/prefect/ingest_topics_gcs.py:ingest_topics_gcs
work_pool: *prod_cloud_run_work_pool
schedule:
cron: "0 9 * * *"
timezone: "Europe/Berlin"
- name: "ingest-photos-napi-gcs-prod"
entrypoint: src/prefect/ingest_photos_napi_gcs.py:ingest_photos_napi_gcs
work_pool: *prod_cloud_run_work_pool
schedule:
cron: "*/60 * * * *" # every N min
timezone: "Europe/Berlin"
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
per_page: 30
proxy_type: datacenter
- name: "ingest-photos-expanded-napi-bigquery-prod"
entrypoint: src/prefect/ingest_photos_expanded_napi_bigquery.py:ingest_photos_expanded_napi_bigquery
work_pool: *prod_cloud_run_work_pool
schedule:
cron: "*/180 * * * *" # every N min
timezone: "Europe/Berlin"
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
proxy_type: datacenter
batch_size: 30
total_record_size: 180
- name: "ingest-photos-gcs-prod"
entrypoint: src/prefect/ingest_photos_gcs.py:ingest_photos_gcs
work_pool: *prod_cloud_run_work_pool
schedule:
cron: "*/60 * * * *" # every N min
timezone: "Europe/Berlin"
parameters:
gcp_credential_block_name: unsplash-photo-trends-deployment-sa
proxy_type: datacenter
batch_size: 20
total_record_size: 60