diff --git a/projects/cloud_ml/Makefile b/projects/cloud_ml/Makefile new file mode 100644 index 0000000000..49de261eb9 --- /dev/null +++ b/projects/cloud_ml/Makefile @@ -0,0 +1,66 @@ +.PHONY: surface_reference_zarr +surface_reference_zarr: + cd scripts && python ./surface_reference_zarr.py + +.PHONY: nudge_to_fine_baseline_run +nudge_to_fine_baseline_run: deploy + cd argo/prognostic-run && ./run.sh nudge-to-fine-baseline + +.PHONY: extend_prognostic_run +extend_prognostic_run: deploy + cd argo/prognostic-run && ./extend.sh $(URL) $(SEGMENTS) + +.PHONY: training_data_zarr +training_data_zarr: + cd scripts && python ./training_data_zarr.py training_data_config.yaml + +.PHONY: prescribed_cloud_cc_decorr_run +prescribed_cloud_cc_decorr_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-cc-decorr + +.PHONY: prescribed_cloud_cc_max_random_run +prescribed_cloud_cc_max_random_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-cc-max-random + +.PHONY: prescribed_cloud_cc_random_run +prescribed_cloud_cc_random_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-cc-random + +.PHONY: prescribed_cloud_decorr_run +prescribed_cloud_decorr_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-decorr + +.PHONY: prescribed_cloud_max_random_run +prescribed_cloud_max_random_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-max-random + +.PHONY: prescribed_cloud_random_run +prescribed_cloud_random_run: deploy + cd argo/prognostic-run && ./run.sh prescribed-cloud-random + +.PHONY: train_cloud_ml_dense_seed +train_cloud_ml_dense_seed: deploy + cd argo/training && ./run_random_seed.sh dense 4 + +.PHONY: upload_squashed_models +upload_squashed_models: + cd scripts/upload-squashed-models && ./upload.sh 4 gs://vcm-ml-experiments/cloud-ml/2023-06-07/train-cloud-ml-dense + +.PHONY: prescribed_cloud_dense_seed_squash_threshold +prescribed_cloud_dense_seed_squash_threshold: deploy + cd argo/prognostic-run && ./run_seed_squash_threshold.sh prescribed-cloud-dense 4 39 + +.PHONY: offline_cloud_predictions_zarr +offline_cloud_predictions_zarr: + cd scripts && python ./offline_predictions_zarr.py offline_predictions_config.yaml + +.PHONY: offline_cloud_predictions_squashed_zarr +offline_cloud_predictions_squashed_zarr: + cd scripts && python ./offline_predictions_zarr.py offline_predictions_squashed_config.yaml + +.PHONY: deploy +deploy: argo/kustomize + cd argo && ./kustomize build . | kubectl apply -f - + +argo/kustomize: + cd argo && ./install_kustomize.sh 3.10.0 \ No newline at end of file diff --git a/projects/cloud_ml/argo/Makefile b/projects/cloud_ml/argo/Makefile deleted file mode 100644 index d019b2d17e..0000000000 --- a/projects/cloud_ml/argo/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -nudge_to_fine_run: deploy - cd nudge-to-fine-run && ./run.sh nudge-to-fine - -nudge_to_fine_run_v2: deploy - cd nudge-to-fine-run && ./run.sh nudge-to-fine-v2 - -nudge_to_fine_run_v3: deploy - cd nudge-to-fine-run && ./run.sh nudge-to-fine-v3 - -nudge_to_fine_run_v4: deploy - cd nudge-to-fine-run && ./run.sh nudge-to-fine-v4 - -extend_nudge_to_fine_run_v4: deploy - cd nudge-to-fine-run && ./extend.sh \ - gs://vcm-ml-experiments/cloud-ml/2022-08-23/cloud-ml-training-data-nudge-to-fine-v4/fv3gfs_run - -baseline_run_v3: deploy - cd nudge-to-fine-run && ./run.sh baseline-v3 - -deploy: kustomize - ./kustomize build . | kubectl apply -f - - -kustomize: - ./install_kustomize.sh 3.10.0 \ No newline at end of file diff --git a/projects/cloud_ml/argo/fine-restarts-to-zarr/pod.yaml b/projects/cloud_ml/argo/fine-restarts-to-zarr/pod.yaml new file mode 100644 index 0000000000..f139e4b1f9 --- /dev/null +++ b/projects/cloud_ml/argo/fine-restarts-to-zarr/pod.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Pod +metadata: + name: fine-restarts-to-zarr +spec: + volumes: + - name: gcp-credentials-user-gcp-sa + secret: + secretName: gcp-key + containers: + - name: main + image: us.gcr.io/vcm-ml/fv3net:400c970a0af572973f5c6b8e970aa261b4baf8a2 + command: ["bash", "-x", "-c"] + workingDir: /home/jovyan/fv3net/workflows/dataflow + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/gcp-credentials/key.json + - name: CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE + value: /secret/gcp-credentials/key.json + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - mountPath: /secret/gcp-credentials + name: gcp-credentials-user-gcp-sa + args: + - | + ./dataflow.sh submit -m fv3net.pipelines.restarts_to_zarr \ + gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts \ + gs://vcm-ml-intermediate/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation-restarts.zarr \ + --no-coarse-suffix \ + --job_name $POD_NAME-$(openssl rand -hex 6) \ + --project vcm-ml \ + --region us-central1 \ + --runner DataFlow \ + --temp_location gs://vcm-ml-scratch/tmp_dataflow \ + --num_workers 64 \ + --autoscaling_algorithm=NONE + resources: + limits: + memory: 2Gi + cpu: "1000m" + requests: + memory: 1Gi + cpu: "1000m" + tolerations: + - effect: NoSchedule + key: dedicated + value: med-sim-pool + restartPolicy: Never \ No newline at end of file diff --git a/projects/cloud_ml/argo/kustomization.yaml b/projects/cloud_ml/argo/kustomization.yaml index 85f93d8b3d..ba96113500 100644 --- a/projects/cloud_ml/argo/kustomization.yaml +++ b/projects/cloud_ml/argo/kustomization.yaml @@ -4,8 +4,10 @@ resources: kind: Kustomization images: - name: us.gcr.io/vcm-ml/fv3net - newTag: 762dfc1e945720f59f5b9e9ab68e10e82a0594a9 + newTag: b9decb4be05133c569a79223b50879e2fd292bd0 - name: us.gcr.io/vcm-ml/post_process_run - newTag: 762dfc1e945720f59f5b9e9ab68e10e82a0594a9 + newTag: b9decb4be05133c569a79223b50879e2fd292bd0 - name: us.gcr.io/vcm-ml/prognostic_run - newTag: 762dfc1e945720f59f5b9e9ab68e10e82a0594a9 \ No newline at end of file + newTag: b9decb4be05133c569a79223b50879e2fd292bd0 +- name: us.gcr.io/vcm-ml/fv3fit + newTag: b9decb4be05133c569a79223b50879e2fd292bd0 \ No newline at end of file diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/extend.sh b/projects/cloud_ml/argo/nudge-to-fine-run/extend.sh deleted file mode 100755 index ccb65cd6da..0000000000 --- a/projects/cloud_ml/argo/nudge-to-fine-run/extend.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -URL=$1 - -NAME="extend-cloud-ml-training-data-$(openssl rand --hex 2)" - -argo submit --from workflowtemplate/restart-prognostic-run \ - -p url="$URL" \ - -p segment-count="4" \ - -p memory="15Gi" \ - --name="$NAME" \ No newline at end of file diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v4-config.yaml b/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v4-config.yaml deleted file mode 100644 index 7cafdf65b2..0000000000 --- a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v4-config.yaml +++ /dev/null @@ -1,320 +0,0 @@ -base_version: v0.7 -prephysics: - - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/C3072-surface-radiative-fluxes-precip.zarr" - variables: - - override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface - - override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface - - override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface - - total_precipitation -initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" -nudging: - restarts_path: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timescale_hours: - air_temperature: 3 - specific_humidity: 3 - x_wind: 3 - y_wind: 3 - pressure_thickness_of_atmospheric_layer: 3 -namelist: - atmos_model_nml: - fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved - coupler_nml: - days: 1 - hours: 0 - minutes: 0 - months: 0 - seconds: 0 - restart_secs: 900 - gfs_physics_nml: - fhlwr: 900.0 - fhswr: 900.0 - override_surface_radiative_fluxes: true - hybedmf: true - satmedmf: false - namsfc: - faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files - faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files - fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files - fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files -diagnostics: -- chunks: - time: 12 - name: state_pre_radiation.zarr - times: - frequency: 900 - kind: interval - variables: - - x_wind_pre_radiation - - y_wind_pre_radiation - - eastward_wind_pre_radiation - - northward_wind_pre_radiation - - vertical_wind_pre_radiation - - pressure_thickness_of_atmospheric_layer_pre_radiation - - vertical_thickness_of_atmospheric_layer_pre_radiation - - atmosphere_hybrid_a_coordinate_pre_radiation - - atmosphere_hybrid_b_coordinate_pre_radiation - - interface_pressure_pre_radiation - - air_temperature_pre_radiation - - specific_humidity_pre_radiation - - cloud_water_mixing_ratio_pre_radiation - - rain_mixing_ratio_pre_radiation - - cloud_ice_mixing_ratio_pre_radiation - - snow_mixing_ratio_pre_radiation - - graupel_mixing_ratio_pre_radiation - - cloud_amount_pre_radiation - - longitude_pre_radiation - - latitude_pre_radiation - - orographic_variables_pre_radiation - - surface_temperature_pre_radiation - - land_sea_mask_pre_radiation - - snow_depth_water_equivalent_pre_radiation - - snow_cover_in_fraction_pre_radiation - - maximum_snow_albedo_in_fraction_pre_radiation - - surface_roughness_pre_radiation - - mean_visible_albedo_with_strong_cosz_dependency_pre_radiation - - mean_near_infrared_albedo_with_strong_cosz_dependency_pre_radiation - - mean_visible_albedo_with_weak_cosz_dependency_pre_radiation - - mean_near_infrared_albedo_with_weak_cosz_dependency_pre_radiation - - fractional_coverage_with_strong_cosz_dependency_pre_radiation - - fractional_coverage_with_weak_cosz_dependency_pre_radiation - - ice_fraction_over_open_water_pre_radiation - - surface_temperature_over_ice_fraction_pre_radiation - - ozone_mixing_ratio_pre_radiation - - clear_sky_downward_longwave_flux_at_surface_pre_radiation - - clear_sky_downward_shortwave_flux_at_surface_pre_radiation - - clear_sky_upward_longwave_flux_at_surface_pre_radiation - - clear_sky_upward_longwave_flux_at_top_of_atmosphere_pre_radiation - - clear_sky_upward_shortwave_flux_at_surface_pre_radiation - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_pre_radiation - - total_sky_downward_longwave_flux_at_surface_pre_radiation - - total_sky_downward_shortwave_flux_at_top_of_atmosphere_pre_radiation - - total_sky_downward_shortwave_flux_at_surface_pre_radiation - - total_sky_upward_longwave_flux_at_surface_pre_radiation - - total_sky_upward_longwave_flux_at_top_of_atmosphere_pre_radiation - - total_sky_upward_shortwave_flux_at_surface_pre_radiation - - total_sky_upward_shortwave_flux_at_top_of_atmosphere_pre_radiation -- chunks: - time: 12 - name: state_after_timestep.zarr - times: - frequency: 900 - kind: interval - variables: - - x_wind - - y_wind - - eastward_wind - - northward_wind - - vertical_wind - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - atmosphere_hybrid_a_coordinate - - atmosphere_hybrid_b_coordinate - - interface_pressure - - air_temperature - - specific_humidity - - cloud_water_mixing_ratio - - rain_mixing_ratio - - cloud_ice_mixing_ratio - - snow_mixing_ratio - - graupel_mixing_ratio - - cloud_amount - - longitude - - latitude - - orographic_variables - - surface_temperature - - land_sea_mask - - snow_depth_water_equivalent - - snow_cover_in_fraction - - maximum_snow_albedo_in_fraction - - surface_roughness - - mean_visible_albedo_with_strong_cosz_dependency - - mean_near_infrared_albedo_with_strong_cosz_dependency - - mean_visible_albedo_with_weak_cosz_dependency - - mean_near_infrared_albedo_with_weak_cosz_dependency - - fractional_coverage_with_strong_cosz_dependency - - fractional_coverage_with_weak_cosz_dependency - - ice_fraction_over_open_water - - surface_temperature_over_ice_fraction - - ozone_mixing_ratio - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_downward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_top_of_atmosphere -- name: physics_tendencies.zarr - chunks: - time: 1 - times: - frequency: 10800 - kind: interval-average - variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics -- name: nudging_tendencies.zarr - chunks: - time: 1 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_tendency_due_to_nudging - - specific_humidity_tendency_due_to_nudging - - x_wind_tendency_due_to_nudging - - y_wind_tendency_due_to_nudging - - pressure_thickness_of_atmospheric_layer_tendency_due_to_nudging -- name: diags.zarr - chunks: - time: 12 - times: - frequency: 900 - kind: interval - variables: - - net_moistening_due_to_nudging - - column_heating_due_to_nudging - - net_mass_tendency_due_to_nudging - - total_precipitation_rate - - water_vapor_path - - physics_precip -- name: reference_state.zarr - chunks: - time: 1 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_reference - - specific_humidity_reference - - x_wind_reference - - y_wind_reference - - pressure_thickness_of_atmospheric_layer_reference -fortran_diagnostics: -- name: sfc_dt_atmos.zarr - chunks: - time: 96 - times: - frequency: 900 - kind: interval - variables: - - {module_name: dynamics, field_name: grid_lont, output_name: lon} - - {module_name: dynamics, field_name: grid_latt, output_name: lat} - - {module_name: dynamics, field_name: grid_lon, output_name: lonb} - - {module_name: dynamics, field_name: grid_lat, output_name: latb} - - {module_name: dynamics, field_name: area, output_name: area} - - {module_name: gfs_phys, field_name: dusfci, output_name: uflx} - - {module_name: gfs_phys, field_name: dvsfci, output_name: vflx} - - {module_name: gfs_phys, field_name: cnvprcpb_ave, output_name: CPRATsfc} - - {module_name: gfs_phys, field_name: totprcpb_ave, output_name: PRATEsfc} - - {module_name: gfs_phys, field_name: toticeb_ave, output_name: ICEsfc} - - {module_name: gfs_phys, field_name: totsnwb_ave, output_name: SNOWsfc} - - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} - - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} - - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} - - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} - - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} - - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} - - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} - - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} - - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} - - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} - - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} - - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} - - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} - - {module_name: gfs_sfc, field_name: fice, output_name: ICECsfc} - - {module_name: gfs_sfc, field_name: SLMSKsfc, output_name: SLMSKsfc} - - {module_name: gfs_sfc, field_name: q2m, output_name: SPFH2m} - - {module_name: gfs_sfc, field_name: t2m, output_name: TMP2m} - - {module_name: gfs_sfc, field_name: tsfc, output_name: TMPsfc} - - {module_name: gfs_phys, field_name: dpt2m, output_name: DPT2m} - - {module_name: gfs_phys, field_name: u10m, output_name: UGRD10m} - - {module_name: gfs_phys, field_name: v10m, output_name: VGRD10m} - - {module_name: gfs_phys, field_name: tmpmax2m, output_name: TMAX2m} - - {module_name: gfs_phys, field_name: wind10mmax, output_name: MAXWIND10m} - - {module_name: gfs_phys, field_name: soilm, output_name: SOILM} - - {module_name: gfs_sfc, field_name: SOILT1, output_name: SOILT1} - - {module_name: gfs_sfc, field_name: SOILT2, output_name: SOILT2} - - {module_name: gfs_sfc, field_name: SOILT3, output_name: SOILT3} - - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} -- name: atmos_dt_atmos.zarr - chunks: - time: 96 - times: - frequency: 900 - kind: interval - variables: - - {module_name: dynamics, field_name: grid_lont, output_name: lon} - - {module_name: dynamics, field_name: grid_latt, output_name: lat} - - {module_name: dynamics, field_name: grid_lon, output_name: lonb} - - {module_name: dynamics, field_name: grid_lat, output_name: latb} - - {module_name: dynamics, field_name: area, output_name: area} - - {module_name: dynamics, field_name: us, output_name: UGRDlowest} - - {module_name: dynamics, field_name: u850, output_name: UGRD850} - - {module_name: dynamics, field_name: u500, output_name: UGRD500} - - {module_name: dynamics, field_name: u200, output_name: UGRD200} - - {module_name: dynamics, field_name: u50, output_name: UGRD50} - - {module_name: dynamics, field_name: vs, output_name: VGRDlowest} - - {module_name: dynamics, field_name: v850, output_name: VGRD850} - - {module_name: dynamics, field_name: v500, output_name: VGRD500} - - {module_name: dynamics, field_name: v200, output_name: VGRD200} - - {module_name: dynamics, field_name: v50, output_name: VGRD50} - - {module_name: dynamics, field_name: tm, output_name: TMP500_300} - - {module_name: dynamics, field_name: tb, output_name: TMPlowest} - - {module_name: dynamics, field_name: t850, output_name: TMP850} - - {module_name: dynamics, field_name: t500, output_name: TMP500} - - {module_name: dynamics, field_name: t200, output_name: TMP200} - - {module_name: dynamics, field_name: w850, output_name: w850} - - {module_name: dynamics, field_name: w500, output_name: w500} - - {module_name: dynamics, field_name: w200, output_name: w200} - - {module_name: dynamics, field_name: w50, output_name: w50} - - {module_name: dynamics, field_name: vort850, output_name: VORT850} - - {module_name: dynamics, field_name: vort500, output_name: VORT500} - - {module_name: dynamics, field_name: vort200, output_name: VORT200} - - {module_name: dynamics, field_name: z850, output_name: h850} - - {module_name: dynamics, field_name: z500, output_name: h500} - - {module_name: dynamics, field_name: z200, output_name: h200} - - {module_name: dynamics, field_name: rh1000, output_name: RH1000} - - {module_name: dynamics, field_name: rh925, output_name: RH925} - - {module_name: dynamics, field_name: rh850, output_name: RH850} - - {module_name: dynamics, field_name: rh700, output_name: RH700} - - {module_name: dynamics, field_name: rh500, output_name: RH500} - - {module_name: dynamics, field_name: rh200, output_name: RH200} - - {module_name: dynamics, field_name: q1000, output_name: q1000} - - {module_name: dynamics, field_name: q925, output_name: q925} - - {module_name: dynamics, field_name: q850, output_name: q850} - - {module_name: dynamics, field_name: q700, output_name: q700} - - {module_name: dynamics, field_name: q500, output_name: q500} - - {module_name: dynamics, field_name: q200, output_name: q200} - - {module_name: dynamics, field_name: slp, output_name: PRMSL} - - {module_name: dynamics, field_name: ps, output_name: PRESsfc} - - {module_name: dynamics, field_name: tq, output_name: PWAT} - - {module_name: dynamics, field_name: lw, output_name: VIL} - - {module_name: dynamics, field_name: iw, output_name: iw} - - {module_name: dynamics, field_name: ke, output_name: kinetic_energy} - - {module_name: dynamics, field_name: te, output_name: total_energy} - - {module_name: dynamics, field_name: column_fv_sat_adj_heating, output_name: column_fv_sat_adj_heating} - - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} -- name: atmos_dt_atmos_3d.zarr - chunks: - time: 24 - times: - frequency: 900 - kind: interval - variables: - - {module_name: dynamics, field_name: grid_lont, output_name: lon} - - {module_name: dynamics, field_name: grid_latt, output_name: lat} - - {module_name: dynamics, field_name: grid_lon, output_name: lonb} - - {module_name: dynamics, field_name: grid_lat, output_name: latb} - - {module_name: dynamics, field_name: fv_sat_adj_qv_dt, output_name: fv_sat_adj_qv_dt} - - {module_name: dynamics, field_name: fv_sat_adj_t_dt, output_name: fv_sat_adj_t_dt} \ No newline at end of file diff --git a/projects/cloud_ml/argo/prognostic-run/extend.sh b/projects/cloud_ml/argo/prognostic-run/extend.sh new file mode 100755 index 0000000000..02e5dee744 --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/extend.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +URL=$1 +SEGMENT_COUNT=$2 + +NAME="cloud-ml-prognostic-run-$(openssl rand --hex 2)" + +argo submit --from workflowtemplate/restart-prognostic-run \ + -p url="$URL" \ + -p segment-count="${SEGMENT_COUNT}" \ + -p cpu="24" \ + -p memory="25Gi" \ + --name="$NAME" \ No newline at end of file diff --git a/projects/cloud_ml/argo/prognostic-run/nudge-to-fine-baseline-config.yaml b/projects/cloud_ml/argo/prognostic-run/nudge-to-fine-baseline-config.yaml new file mode 100644 index 0000000000..a5d99343a9 --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/nudge-to-fine-baseline-config.yaml @@ -0,0 +1,584 @@ +base_version: v0.7 +prephysics: + - dataset_key: gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr + variables: + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation +initial_conditions: + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: '20200731.001500' +nudging: + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timescale_hours: + air_temperature: 3 + specific_humidity: 3 + x_wind: 3 + y_wind: 3 + pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: python +namelist: + atmos_model_nml: + fhmax: 1024 + coupler_nml: + days: 0 + hours: 6 + minutes: 0 + months: 0 + seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150 + gfs_physics_nml: + fhzero: 0.25 + fhlwr: 900 + fhswr: 900 + iovr_sw: 3 + iovr_lw: 3 + ccnorm: true + override_surface_radiative_fluxes: true + hybedmf: true + satmedmf: false + xkzm_h: 2 + fv_core_nml: + layout: + - 2 + - 2 + namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology + faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files + fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files +diagnostics: + - name: state_pre_radiation.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - x_wind_pre_radiation + - y_wind_pre_radiation + - eastward_wind_pre_radiation + - northward_wind_pre_radiation + - vertical_wind_pre_radiation + - air_temperature_pre_radiation + - specific_humidity_pre_radiation + - pressure_thickness_of_atmospheric_layer_pre_radiation + - vertical_thickness_of_atmospheric_layer_pre_radiation + - land_sea_mask_pre_radiation + - surface_temperature_pre_radiation + - surface_geopotential_pre_radiation + - surface_diffused_shortwave_albedo_pre_radiation + - sensible_heat_flux_pre_radiation + - latent_heat_flux_pre_radiation + - total_precipitation_pre_radiation + - surface_precipitation_rate_pre_radiation + - total_soil_moisture_pre_radiation + - cloud_water_mixing_ratio_pre_radiation + - rain_mixing_ratio_pre_radiation + - cloud_ice_mixing_ratio_pre_radiation + - snow_mixing_ratio_pre_radiation + - graupel_mixing_ratio_pre_radiation + - cloud_amount_pre_radiation + - name: state_after_timestep.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - x_wind + - y_wind + - eastward_wind + - northward_wind + - vertical_wind + - air_temperature + - specific_humidity + - pressure_thickness_of_atmospheric_layer + - vertical_thickness_of_atmospheric_layer + - land_sea_mask + - surface_temperature + - surface_geopotential + - surface_diffused_shortwave_albedo + - sensible_heat_flux + - latent_heat_flux + - total_precipitation + - surface_precipitation_rate + - total_soil_moisture + - latitude + - longitude + - cloud_water_mixing_ratio + - cloud_ice_mixing_ratio + - rain_mixing_ratio + - snow_mixing_ratio + - graupel_mixing_ratio + - cloud_amount + - name: radiative_fluxes.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python + - name: physics_tendencies.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - tendency_of_air_temperature_due_to_fv3_physics + - tendency_of_specific_humidity_due_to_fv3_physics + - tendency_of_eastward_wind_due_to_fv3_physics + - tendency_of_northward_wind_due_to_fv3_physics + - name: nudging_tendencies.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - air_temperature_tendency_due_to_nudging + - specific_humidity_tendency_due_to_nudging + - x_wind_tendency_due_to_nudging + - y_wind_tendency_due_to_nudging + - pressure_thickness_of_atmospheric_layer_tendency_due_to_nudging + - name: diags.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - net_moistening_due_to_nudging + - column_heating_due_to_nudging + - net_mass_tendency_due_to_nudging + - total_precipitation_rate + - water_vapor_path + - physics_precip + - name: reference_state.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - air_temperature_reference + - specific_humidity_reference + - x_wind_reference + - y_wind_reference + - pressure_thickness_of_atmospheric_layer_reference +fortran_diagnostics: + - name: sfc_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - module_name: dynamics + field_name: grid_lont + output_name: lon + - module_name: dynamics + field_name: grid_latt + output_name: lat + - module_name: dynamics + field_name: grid_lon + output_name: lonb + - module_name: dynamics + field_name: grid_lat + output_name: latb + - module_name: dynamics + field_name: area + output_name: area + - module_name: gfs_phys + field_name: dusfci + output_name: uflx + - module_name: gfs_phys + field_name: dvsfci + output_name: vflx + - module_name: gfs_phys + field_name: cnvprcpb_ave + output_name: CPRATsfc + - module_name: gfs_phys + field_name: totprcpb_ave + output_name: PRATEsfc + - module_name: gfs_phys + field_name: toticeb_ave + output_name: ICEsfc + - module_name: gfs_phys + field_name: totsnwb_ave + output_name: SNOWsfc + - module_name: gfs_phys + field_name: totgrpb_ave + output_name: GRAUPELsfc + - module_name: gfs_phys + field_name: DSWRF + output_name: DSWRFsfc + - module_name: gfs_phys + field_name: DSWRF_from_rrtmg + output_name: DSWRFsfc_from_RRTMG + - module_name: gfs_phys + field_name: DSWRFI + output_name: DSWRFIsfc + - module_name: gfs_phys + field_name: csdsf_ave + output_name: DCSWRFsfc + - module_name: gfs_phys + field_name: USWRF + output_name: USWRFsfc + - module_name: gfs_phys + field_name: USWRF_from_rrtmg + output_name: USWRFsfc_from_RRTMG + - module_name: gfs_phys + field_name: USWRFI + output_name: USWRFIsfc + - module_name: gfs_phys + field_name: csusf_ave + output_name: UCSWRFsfc + - module_name: gfs_phys + field_name: DSWRFtoa + output_name: DSWRFtoa + - module_name: gfs_phys + field_name: sw_dnfxc + output_name: DSWRFItoa + - module_name: gfs_phys + field_name: USWRFtoa + output_name: USWRFtoa + - module_name: gfs_phys + field_name: sw_upfxc + output_name: USWRFItoa + - module_name: gfs_phys + field_name: csusf_avetoa + output_name: UCSWRFtoa + - module_name: gfs_phys + field_name: ULWRFtoa + output_name: ULWRFtoa + - module_name: gfs_phys + field_name: lw_upfxc + output_name: ULWRFItoa + - module_name: gfs_phys + field_name: csulf_avetoa + output_name: UCLWRFtoa + - module_name: gfs_phys + field_name: ULWRF + output_name: ULWRFsfc + - module_name: gfs_phys + field_name: ULWRFI + output_name: ULWRFIsfc + - module_name: gfs_phys + field_name: csulf_ave + output_name: UCLWRFsfc + - module_name: gfs_phys + field_name: DLWRF + output_name: DLWRFsfc + - module_name: gfs_phys + field_name: DLWRF_from_rrtmg + output_name: DLWRFsfc_from_RRTMG + - module_name: gfs_phys + field_name: DLWRFI + output_name: DLWRFIsfc + - module_name: gfs_phys + field_name: csdlf_ave + output_name: DCLWRFsfc + - module_name: gfs_phys + field_name: lhtfl_ave + output_name: LHTFLsfc + - module_name: gfs_phys + field_name: shtfl_ave + output_name: SHTFLsfc + - module_name: gfs_phys + field_name: hpbl + output_name: HPBLsfc + - module_name: gfs_sfc + field_name: fice + output_name: ICECsfc + - module_name: gfs_sfc + field_name: SLMSKsfc + output_name: SLMSKsfc + - module_name: gfs_sfc + field_name: q2m + output_name: SPFH2m + - module_name: gfs_sfc + field_name: t2m + output_name: TMP2m + - module_name: gfs_sfc + field_name: tsfc + output_name: TMPsfc + - module_name: gfs_phys + field_name: dpt2m + output_name: DPT2m + - module_name: gfs_phys + field_name: u10m + output_name: UGRD10m + - module_name: gfs_phys + field_name: v10m + output_name: VGRD10m + - module_name: gfs_phys + field_name: tmpmax2m + output_name: TMAX2m + - module_name: gfs_phys + field_name: wind10mmax + output_name: MAXWIND10m + - module_name: gfs_phys + field_name: snowc_ave + output_name: SNOWCA + - module_name: gfs_sfc + field_name: snowd + output_name: SNOWD + - module_name: gfs_sfc + field_name: snoalb + output_name: SNOWALB + - module_name: gfs_phys + field_name: soilm + output_name: SOILM + - module_name: gfs_sfc + field_name: SOILT1 + output_name: SOILT1 + - module_name: gfs_sfc + field_name: SOILT2 + output_name: SOILT2 + - module_name: gfs_sfc + field_name: SOILT3 + output_name: SOILT3 + - module_name: gfs_sfc + field_name: SOILT4 + output_name: SOILT4 + - name: atmos_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - module_name: dynamics + field_name: grid_lont + output_name: lon + - module_name: dynamics + field_name: grid_latt + output_name: lat + - module_name: dynamics + field_name: grid_lon + output_name: lonb + - module_name: dynamics + field_name: grid_lat + output_name: latb + - module_name: dynamics + field_name: area + output_name: area + - module_name: dynamics + field_name: us + output_name: UGRDlowest + - module_name: dynamics + field_name: u850 + output_name: UGRD850 + - module_name: dynamics + field_name: u500 + output_name: UGRD500 + - module_name: dynamics + field_name: u200 + output_name: UGRD200 + - module_name: dynamics + field_name: u50 + output_name: UGRD50 + - module_name: dynamics + field_name: vs + output_name: VGRDlowest + - module_name: dynamics + field_name: v850 + output_name: VGRD850 + - module_name: dynamics + field_name: v500 + output_name: VGRD500 + - module_name: dynamics + field_name: v200 + output_name: VGRD200 + - module_name: dynamics + field_name: v50 + output_name: VGRD50 + - module_name: dynamics + field_name: tm + output_name: TMP500_300 + - module_name: dynamics + field_name: tb + output_name: TMPlowest + - module_name: dynamics + field_name: t850 + output_name: TMP850 + - module_name: dynamics + field_name: t500 + output_name: TMP500 + - module_name: dynamics + field_name: t200 + output_name: TMP200 + - module_name: dynamics + field_name: w850 + output_name: w850 + - module_name: dynamics + field_name: w500 + output_name: w500 + - module_name: dynamics + field_name: w200 + output_name: w200 + - module_name: dynamics + field_name: w50 + output_name: w50 + - module_name: dynamics + field_name: vort850 + output_name: VORT850 + - module_name: dynamics + field_name: vort500 + output_name: VORT500 + - module_name: dynamics + field_name: vort200 + output_name: VORT200 + - module_name: dynamics + field_name: z850 + output_name: h850 + - module_name: dynamics + field_name: z500 + output_name: h500 + - module_name: dynamics + field_name: z200 + output_name: h200 + - module_name: dynamics + field_name: rh1000 + output_name: RH1000 + - module_name: dynamics + field_name: rh925 + output_name: RH925 + - module_name: dynamics + field_name: rh850 + output_name: RH850 + - module_name: dynamics + field_name: rh700 + output_name: RH700 + - module_name: dynamics + field_name: rh500 + output_name: RH500 + - module_name: dynamics + field_name: rh200 + output_name: RH200 + - module_name: dynamics + field_name: q1000 + output_name: q1000 + - module_name: dynamics + field_name: q925 + output_name: q925 + - module_name: dynamics + field_name: q850 + output_name: q850 + - module_name: dynamics + field_name: q700 + output_name: q700 + - module_name: dynamics + field_name: q500 + output_name: q500 + - module_name: dynamics + field_name: q200 + output_name: q200 + - module_name: dynamics + field_name: slp + output_name: PRMSL + - module_name: dynamics + field_name: ps + output_name: PRESsfc + - module_name: dynamics + field_name: tq + output_name: PWAT + - module_name: dynamics + field_name: lw + output_name: VIL + - module_name: dynamics + field_name: iw + output_name: iw + - module_name: dynamics + field_name: ke + output_name: kinetic_energy + - module_name: dynamics + field_name: te + output_name: total_energy + - module_name: dynamics + field_name: column_fv_sat_adj_heating + output_name: column_fv_sat_adj_heating + - module_name: dynamics + field_name: column_fv_sat_adj_moistening + output_name: column_fv_sat_adj_moistening + - name: atmos_dt_atmos_3d.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - module_name: dynamics + field_name: grid_lont + output_name: lon + - module_name: dynamics + field_name: grid_latt + output_name: lat + - module_name: dynamics + field_name: grid_lon + output_name: lonb + - module_name: dynamics + field_name: grid_lat + output_name: latb + - module_name: dynamics + field_name: fv_sat_adj_qv_dt + output_name: fv_sat_adj_qv_dt + - module_name: dynamics + field_name: fv_sat_adj_t_dt + output_name: fv_sat_adj_t_dt + - name: physics_tendencies_3d.zarr + chunks: + time: 1 + times: + frequency: 900 + kind: interval + variables: + - module_name: gfs_phys + field_name: tendency_of_air_temperature_due_to_shortwave_heating + output_name: tendency_of_air_temperature_due_to_shortwave_heating + - module_name: gfs_phys + field_name: tendency_of_air_temperature_due_to_longwave_heating + output_name: tendency_of_air_temperature_due_to_longwave_heating + - module_name: gfs_phys + field_name: tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky + output_name: tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky + - module_name: gfs_phys + field_name: tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky + output_name: tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky + - module_name: gfs_phys + field_name: vertically_integrated_tendency_of_air_temperature_due_to_shortwave_heating + output_name: vertically_integrated_tendency_of_air_temperature_due_to_shortwave_heating + - module_name: gfs_phys + field_name: vertically_integrated_tendency_of_air_temperature_due_to_longwave_heating + output_name: vertically_integrated_tendency_of_air_temperature_due_to_longwave_heating + - module_name: gfs_phys + field_name: vertically_integrated_tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky + output_name: vertically_integrated_tendency_of_air_temperature_due_to_shortwave_heating_assuming_clear_sky + - module_name: gfs_phys + field_name: vertically_integrated_tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky + output_name: vertically_integrated_tendency_of_air_temperature_due_to_longwave_heating_assuming_clear_sky diff --git a/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-decorr-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-decorr-config.yaml new file mode 100644 index 0000000000..f09ccd36bd --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-decorr-config.yaml @@ -0,0 +1,236 @@ +base_version: v0.7 +prephysics: + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" + variables: + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation +initial_conditions: + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" +nudging: + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timescale_hours: + air_temperature: 3 + specific_humidity: 3 + x_wind: 3 + y_wind: 3 + pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 +namelist: + atmos_model_nml: + fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved + coupler_nml: + days: 0 + hours: 6 + minutes: 0 + months: 0 + seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 + gfs_physics_nml: + fhzero: 0.25 + fhlwr: 900.0 + fhswr: 900.0 + iovr_sw: 3 + iovr_lw: 3 + ccnorm: True + override_surface_radiative_fluxes: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 + namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology + faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files + fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files +diagnostics: +- name: radiative_fluxes.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python +- name: diags.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - net_moistening_due_to_nudging + - column_heating_due_to_nudging + - net_mass_tendency_due_to_nudging + - total_precipitation_rate + - water_vapor_path + - physics_precip +fortran_diagnostics: +- name: sfc_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: gfs_phys, field_name: dusfci, output_name: uflx} + - {module_name: gfs_phys, field_name: dvsfci, output_name: vflx} + - {module_name: gfs_phys, field_name: cnvprcpb_ave, output_name: CPRATsfc} + - {module_name: gfs_phys, field_name: totprcpb_ave, output_name: PRATEsfc} + - {module_name: gfs_phys, field_name: toticeb_ave, output_name: ICEsfc} + - {module_name: gfs_phys, field_name: totsnwb_ave, output_name: SNOWsfc} + - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} + - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} + - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} + - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} + - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} + - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} + - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} + - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} + - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} + - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} + - {module_name: gfs_sfc, field_name: fice, output_name: ICECsfc} + - {module_name: gfs_sfc, field_name: SLMSKsfc, output_name: SLMSKsfc} + - {module_name: gfs_sfc, field_name: q2m, output_name: SPFH2m} + - {module_name: gfs_sfc, field_name: t2m, output_name: TMP2m} + - {module_name: gfs_sfc, field_name: tsfc, output_name: TMPsfc} + - {module_name: gfs_phys, field_name: dpt2m, output_name: DPT2m} + - {module_name: gfs_phys, field_name: u10m, output_name: UGRD10m} + - {module_name: gfs_phys, field_name: v10m, output_name: VGRD10m} + - {module_name: gfs_phys, field_name: tmpmax2m, output_name: TMAX2m} + - {module_name: gfs_phys, field_name: wind10mmax, output_name: MAXWIND10m} + - {module_name: gfs_phys, field_name: snowc_ave, output_name: SNOWCA} + - {module_name: gfs_sfc, field_name: snowd, output_name: SNOWD} + - {module_name: gfs_sfc, field_name: snoalb, output_name: SNOWALB} + - {module_name: gfs_phys, field_name: soilm, output_name: SOILM} + - {module_name: gfs_sfc, field_name: SOILT1, output_name: SOILT1} + - {module_name: gfs_sfc, field_name: SOILT2, output_name: SOILT2} + - {module_name: gfs_sfc, field_name: SOILT3, output_name: SOILT3} + - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} +- name: atmos_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: dynamics, field_name: us, output_name: UGRDlowest} + - {module_name: dynamics, field_name: u850, output_name: UGRD850} + - {module_name: dynamics, field_name: u500, output_name: UGRD500} + - {module_name: dynamics, field_name: u200, output_name: UGRD200} + - {module_name: dynamics, field_name: u50, output_name: UGRD50} + - {module_name: dynamics, field_name: vs, output_name: VGRDlowest} + - {module_name: dynamics, field_name: v850, output_name: VGRD850} + - {module_name: dynamics, field_name: v500, output_name: VGRD500} + - {module_name: dynamics, field_name: v200, output_name: VGRD200} + - {module_name: dynamics, field_name: v50, output_name: VGRD50} + - {module_name: dynamics, field_name: tm, output_name: TMP500_300} + - {module_name: dynamics, field_name: tb, output_name: TMPlowest} + - {module_name: dynamics, field_name: t850, output_name: TMP850} + - {module_name: dynamics, field_name: t500, output_name: TMP500} + - {module_name: dynamics, field_name: t200, output_name: TMP200} + - {module_name: dynamics, field_name: w850, output_name: w850} + - {module_name: dynamics, field_name: w500, output_name: w500} + - {module_name: dynamics, field_name: w200, output_name: w200} + - {module_name: dynamics, field_name: w50, output_name: w50} + - {module_name: dynamics, field_name: vort850, output_name: VORT850} + - {module_name: dynamics, field_name: vort500, output_name: VORT500} + - {module_name: dynamics, field_name: vort200, output_name: VORT200} + - {module_name: dynamics, field_name: z850, output_name: h850} + - {module_name: dynamics, field_name: z500, output_name: h500} + - {module_name: dynamics, field_name: z200, output_name: h200} + - {module_name: dynamics, field_name: rh1000, output_name: RH1000} + - {module_name: dynamics, field_name: rh925, output_name: RH925} + - {module_name: dynamics, field_name: rh850, output_name: RH850} + - {module_name: dynamics, field_name: rh700, output_name: RH700} + - {module_name: dynamics, field_name: rh500, output_name: RH500} + - {module_name: dynamics, field_name: rh200, output_name: RH200} + - {module_name: dynamics, field_name: q1000, output_name: q1000} + - {module_name: dynamics, field_name: q925, output_name: q925} + - {module_name: dynamics, field_name: q850, output_name: q850} + - {module_name: dynamics, field_name: q700, output_name: q700} + - {module_name: dynamics, field_name: q500, output_name: q500} + - {module_name: dynamics, field_name: q200, output_name: q200} + - {module_name: dynamics, field_name: slp, output_name: PRMSL} + - {module_name: dynamics, field_name: ps, output_name: PRESsfc} + - {module_name: dynamics, field_name: tq, output_name: PWAT} + - {module_name: dynamics, field_name: lw, output_name: VIL} + - {module_name: dynamics, field_name: iw, output_name: iw} + - {module_name: dynamics, field_name: ke, output_name: kinetic_energy} + - {module_name: dynamics, field_name: te, output_name: total_energy} + - {module_name: dynamics, field_name: column_fv_sat_adj_heating, output_name: column_fv_sat_adj_heating} + - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} +- name: atmos_dt_atmos_3d.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: fv_sat_adj_qv_dt, output_name: fv_sat_adj_qv_dt} + - {module_name: dynamics, field_name: fv_sat_adj_t_dt, output_name: fv_sat_adj_t_dt} \ No newline at end of file diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v3-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-max-random-config.yaml similarity index 70% rename from projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v3-config.yaml rename to projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-max-random-config.yaml index 23bd834515..4c4473732b 100644 --- a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v3-config.yaml +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-max-random-config.yaml @@ -1,118 +1,94 @@ base_version: v0.7 -field_table: gs://vcm-fv3config/config/field_table/TKE-EDMF/v1.0/field_table prephysics: - - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/C3072-surface-radiative-fluxes-precip.zarr" + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" variables: - - override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface - - override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface - - override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface - - total_precipitation + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" nudging: - restarts_path: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts timescale_hours: air_temperature: 3 specific_humidity: 3 x_wind: 3 y_wind: 3 pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 namelist: atmos_model_nml: fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved coupler_nml: - days: 1 - hours: 0 + days: 0 + hours: 6 minutes: 0 months: 0 seconds: 0 - restart_secs: 900 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 gfs_physics_nml: + fhzero: 0.25 fhlwr: 900.0 fhswr: 900.0 + iovr_sw: 1 + iovr_lw: 1 + ccnorm: True override_surface_radiative_fluxes: true - hybedmf: false - satmedmf: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files diagnostics: -- name: state_after_timestep.zarr - chunks: - time: 24 - times: - frequency: 900 - kind: interval - variables: - - x_wind - - y_wind - - eastward_wind - - northward_wind - - vertical_wind - - air_temperature - - specific_humidity - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - land_sea_mask - - surface_temperature - - surface_geopotential - - surface_diffused_shortwave_albedo - - sensible_heat_flux - - latent_heat_flux - - total_precipitation - - surface_precipitation_rate - - total_soil_moisture - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_surface - - total_sky_downward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - latitude - - longitude - - cloud_water_mixing_ratio - - cloud_ice_mixing_ratio - - rain_mixing_ratio - - snow_mixing_ratio - - graupel_mixing_ratio - - cloud_amount - - orographic_variables -- name: physics_tendencies.zarr +- name: radiative_fluxes.zarr chunks: - time: 24 + time: 4 times: frequency: 900 kind: interval variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics -- name: nudging_tendencies.zarr - chunks: - time: 24 - times: - frequency: 900 - kind: interval-average - variables: - - air_temperature_tendency_due_to_nudging - - specific_humidity_tendency_due_to_nudging - - x_wind_tendency_due_to_nudging - - y_wind_tendency_due_to_nudging - - pressure_thickness_of_atmospheric_layer_tendency_due_to_nudging + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python - name: diags.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -123,22 +99,10 @@ diagnostics: - total_precipitation_rate - water_vapor_path - physics_precip -- name: reference_state.zarr - chunks: - time: 24 - times: - frequency: 900 - kind: interval-average - variables: - - air_temperature_reference - - specific_humidity_reference - - x_wind_reference - - y_wind_reference - - pressure_thickness_of_atmospheric_layer_reference fortran_diagnostics: - name: sfc_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -157,14 +121,27 @@ fortran_diagnostics: - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} @@ -185,7 +162,7 @@ fortran_diagnostics: - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} - name: atmos_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -243,9 +220,9 @@ fortran_diagnostics: - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} - name: atmos_dt_atmos_3d.zarr chunks: - time: 24 + time: 1 times: - frequency: 900 + frequency: 3600 kind: interval variables: - {module_name: dynamics, field_name: grid_lont, output_name: lon} diff --git a/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-random-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-random-config.yaml new file mode 100644 index 0000000000..6bc5109557 --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-cc-random-config.yaml @@ -0,0 +1,233 @@ +base_version: v0.7 +prephysics: + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" + variables: + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation +initial_conditions: + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" +nudging: + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timescale_hours: + air_temperature: 3 + specific_humidity: 3 + x_wind: 3 + y_wind: 3 + pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 +namelist: + atmos_model_nml: + fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved + coupler_nml: + days: 0 + hours: 6 + minutes: 0 + months: 0 + seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 + gfs_physics_nml: + fhzero: 0.25 + fhlwr: 900.0 + fhswr: 900.0 + iovr_sw: 0 + iovr_lw: 0 + ccnorm: True + override_surface_radiative_fluxes: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 + namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology + faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files + fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files +diagnostics: +- name: radiative_fluxes.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python +- name: diags.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - net_moistening_due_to_nudging + - column_heating_due_to_nudging + - net_mass_tendency_due_to_nudging + - total_precipitation_rate + - water_vapor_path + - physics_precip +fortran_diagnostics: +- name: sfc_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: gfs_phys, field_name: dusfci, output_name: uflx} + - {module_name: gfs_phys, field_name: dvsfci, output_name: vflx} + - {module_name: gfs_phys, field_name: cnvprcpb_ave, output_name: CPRATsfc} + - {module_name: gfs_phys, field_name: totprcpb_ave, output_name: PRATEsfc} + - {module_name: gfs_phys, field_name: toticeb_ave, output_name: ICEsfc} + - {module_name: gfs_phys, field_name: totsnwb_ave, output_name: SNOWsfc} + - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} + - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} + - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} + - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} + - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} + - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} + - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} + - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} + - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} + - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} + - {module_name: gfs_sfc, field_name: fice, output_name: ICECsfc} + - {module_name: gfs_sfc, field_name: SLMSKsfc, output_name: SLMSKsfc} + - {module_name: gfs_sfc, field_name: q2m, output_name: SPFH2m} + - {module_name: gfs_sfc, field_name: t2m, output_name: TMP2m} + - {module_name: gfs_sfc, field_name: tsfc, output_name: TMPsfc} + - {module_name: gfs_phys, field_name: dpt2m, output_name: DPT2m} + - {module_name: gfs_phys, field_name: u10m, output_name: UGRD10m} + - {module_name: gfs_phys, field_name: v10m, output_name: VGRD10m} + - {module_name: gfs_phys, field_name: tmpmax2m, output_name: TMAX2m} + - {module_name: gfs_phys, field_name: wind10mmax, output_name: MAXWIND10m} + - {module_name: gfs_phys, field_name: soilm, output_name: SOILM} + - {module_name: gfs_sfc, field_name: SOILT1, output_name: SOILT1} + - {module_name: gfs_sfc, field_name: SOILT2, output_name: SOILT2} + - {module_name: gfs_sfc, field_name: SOILT3, output_name: SOILT3} + - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} +- name: atmos_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: dynamics, field_name: us, output_name: UGRDlowest} + - {module_name: dynamics, field_name: u850, output_name: UGRD850} + - {module_name: dynamics, field_name: u500, output_name: UGRD500} + - {module_name: dynamics, field_name: u200, output_name: UGRD200} + - {module_name: dynamics, field_name: u50, output_name: UGRD50} + - {module_name: dynamics, field_name: vs, output_name: VGRDlowest} + - {module_name: dynamics, field_name: v850, output_name: VGRD850} + - {module_name: dynamics, field_name: v500, output_name: VGRD500} + - {module_name: dynamics, field_name: v200, output_name: VGRD200} + - {module_name: dynamics, field_name: v50, output_name: VGRD50} + - {module_name: dynamics, field_name: tm, output_name: TMP500_300} + - {module_name: dynamics, field_name: tb, output_name: TMPlowest} + - {module_name: dynamics, field_name: t850, output_name: TMP850} + - {module_name: dynamics, field_name: t500, output_name: TMP500} + - {module_name: dynamics, field_name: t200, output_name: TMP200} + - {module_name: dynamics, field_name: w850, output_name: w850} + - {module_name: dynamics, field_name: w500, output_name: w500} + - {module_name: dynamics, field_name: w200, output_name: w200} + - {module_name: dynamics, field_name: w50, output_name: w50} + - {module_name: dynamics, field_name: vort850, output_name: VORT850} + - {module_name: dynamics, field_name: vort500, output_name: VORT500} + - {module_name: dynamics, field_name: vort200, output_name: VORT200} + - {module_name: dynamics, field_name: z850, output_name: h850} + - {module_name: dynamics, field_name: z500, output_name: h500} + - {module_name: dynamics, field_name: z200, output_name: h200} + - {module_name: dynamics, field_name: rh1000, output_name: RH1000} + - {module_name: dynamics, field_name: rh925, output_name: RH925} + - {module_name: dynamics, field_name: rh850, output_name: RH850} + - {module_name: dynamics, field_name: rh700, output_name: RH700} + - {module_name: dynamics, field_name: rh500, output_name: RH500} + - {module_name: dynamics, field_name: rh200, output_name: RH200} + - {module_name: dynamics, field_name: q1000, output_name: q1000} + - {module_name: dynamics, field_name: q925, output_name: q925} + - {module_name: dynamics, field_name: q850, output_name: q850} + - {module_name: dynamics, field_name: q700, output_name: q700} + - {module_name: dynamics, field_name: q500, output_name: q500} + - {module_name: dynamics, field_name: q200, output_name: q200} + - {module_name: dynamics, field_name: slp, output_name: PRMSL} + - {module_name: dynamics, field_name: ps, output_name: PRESsfc} + - {module_name: dynamics, field_name: tq, output_name: PWAT} + - {module_name: dynamics, field_name: lw, output_name: VIL} + - {module_name: dynamics, field_name: iw, output_name: iw} + - {module_name: dynamics, field_name: ke, output_name: kinetic_energy} + - {module_name: dynamics, field_name: te, output_name: total_energy} + - {module_name: dynamics, field_name: column_fv_sat_adj_heating, output_name: column_fv_sat_adj_heating} + - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} +- name: atmos_dt_atmos_3d.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: fv_sat_adj_qv_dt, output_name: fv_sat_adj_qv_dt} + - {module_name: dynamics, field_name: fv_sat_adj_t_dt, output_name: fv_sat_adj_t_dt} diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v2-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-decorr-config.yaml similarity index 69% rename from projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v2-config.yaml rename to projects/cloud_ml/argo/prognostic-run/prescribed-cloud-decorr-config.yaml index 519c5015bb..32fc4ca2b7 100644 --- a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-v2-config.yaml +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-decorr-config.yaml @@ -1,117 +1,94 @@ base_version: v0.7 -field_table: gs://vcm-fv3config/config/field_table/TKE-EDMF/v1.0/field_table prephysics: - - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/C3072-surface-radiative-fluxes-precip.zarr" + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" variables: - - override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface - - override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface - - override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface - - total_precipitation + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" nudging: - restarts_path: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts timescale_hours: air_temperature: 3 specific_humidity: 3 x_wind: 3 y_wind: 3 pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 namelist: atmos_model_nml: fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved coupler_nml: - days: 5 - hours: 0 + days: 0 + hours: 6 minutes: 0 months: 0 seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 gfs_physics_nml: - fhlwr: 1800.0 - fhswr: 1800.0 + fhzero: 0.25 + fhlwr: 900.0 + fhswr: 900.0 + iovr_sw: 3 + iovr_lw: 3 + ccnorm: false override_surface_radiative_fluxes: true - hybedmf: false - satmedmf: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files diagnostics: -- name: state_after_timestep.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval - variables: - - x_wind - - y_wind - - eastward_wind - - northward_wind - - vertical_wind - - air_temperature - - specific_humidity - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - land_sea_mask - - surface_temperature - - surface_geopotential - - surface_diffused_shortwave_albedo - - sensible_heat_flux - - latent_heat_flux - - total_precipitation - - surface_precipitation_rate - - total_soil_moisture - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_surface - - total_sky_downward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - latitude - - longitude - - cloud_water_mixing_ratio - - cloud_ice_mixing_ratio - - rain_mixing_ratio - - snow_mixing_ratio - - graupel_mixing_ratio - - cloud_amount - - orographic_variables -- name: physics_tendencies.zarr +- name: radiative_fluxes.zarr chunks: - time: 16 + time: 4 times: - frequency: 5400 + frequency: 900 kind: interval variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics -- name: nudging_tendencies.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_tendency_due_to_nudging - - specific_humidity_tendency_due_to_nudging - - x_wind_tendency_due_to_nudging - - y_wind_tendency_due_to_nudging - - pressure_thickness_of_atmospheric_layer_tendency_due_to_nudging + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python - name: diags.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -122,22 +99,10 @@ diagnostics: - total_precipitation_rate - water_vapor_path - physics_precip -- name: reference_state.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_reference - - specific_humidity_reference - - x_wind_reference - - y_wind_reference - - pressure_thickness_of_atmospheric_layer_reference fortran_diagnostics: - name: sfc_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -156,14 +121,27 @@ fortran_diagnostics: - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} @@ -184,7 +162,7 @@ fortran_diagnostics: - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} - name: atmos_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -242,9 +220,9 @@ fortran_diagnostics: - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} - name: atmos_dt_atmos_3d.zarr chunks: - time: 16 + time: 1 times: - frequency: 5400 + frequency: 3600 kind: interval variables: - {module_name: dynamics, field_name: grid_lont, output_name: lon} diff --git a/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-dense-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-dense-config.yaml new file mode 100644 index 0000000000..51a5e11d51 --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-dense-config.yaml @@ -0,0 +1,233 @@ +base_version: v0.7 +prephysics: + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" + variables: + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation +initial_conditions: + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" +nudging: + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timescale_hours: + air_temperature: 3 + specific_humidity: 3 + x_wind: 3 + y_wind: 3 + pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + model: + - gs://vcm-ml-experiments/cloud-ml/2023-06-07/train-cloud-ml-dense/squashed-models/dense-seed-0/squash-threshold-0 + input_standard_names: + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio_coarse +namelist: + atmos_model_nml: + fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved + coupler_nml: + days: 0 + hours: 6 + minutes: 0 + months: 0 + seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 + gfs_physics_nml: + fhzero: 0.25 + fhlwr: 900.0 + fhswr: 900.0 + iovr_sw: 3 + iovr_lw: 3 + ccnorm: True + override_surface_radiative_fluxes: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 + namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology + faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files + fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files + fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files +diagnostics: +- name: radiative_fluxes.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python +- name: diags.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - net_moistening_due_to_nudging + - column_heating_due_to_nudging + - net_mass_tendency_due_to_nudging + - total_precipitation_rate + - water_vapor_path + - physics_precip +fortran_diagnostics: +- name: sfc_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: gfs_phys, field_name: dusfci, output_name: uflx} + - {module_name: gfs_phys, field_name: dvsfci, output_name: vflx} + - {module_name: gfs_phys, field_name: cnvprcpb_ave, output_name: CPRATsfc} + - {module_name: gfs_phys, field_name: totprcpb_ave, output_name: PRATEsfc} + - {module_name: gfs_phys, field_name: toticeb_ave, output_name: ICEsfc} + - {module_name: gfs_phys, field_name: totsnwb_ave, output_name: SNOWsfc} + - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} + - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} + - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} + - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} + - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} + - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} + - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} + - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} + - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} + - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} + - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} + - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} + - {module_name: gfs_sfc, field_name: fice, output_name: ICECsfc} + - {module_name: gfs_sfc, field_name: SLMSKsfc, output_name: SLMSKsfc} + - {module_name: gfs_sfc, field_name: q2m, output_name: SPFH2m} + - {module_name: gfs_sfc, field_name: t2m, output_name: TMP2m} + - {module_name: gfs_sfc, field_name: tsfc, output_name: TMPsfc} + - {module_name: gfs_phys, field_name: dpt2m, output_name: DPT2m} + - {module_name: gfs_phys, field_name: u10m, output_name: UGRD10m} + - {module_name: gfs_phys, field_name: v10m, output_name: VGRD10m} + - {module_name: gfs_phys, field_name: tmpmax2m, output_name: TMAX2m} + - {module_name: gfs_phys, field_name: wind10mmax, output_name: MAXWIND10m} + - {module_name: gfs_phys, field_name: snowc_ave, output_name: SNOWCA} + - {module_name: gfs_sfc, field_name: snowd, output_name: SNOWD} + - {module_name: gfs_sfc, field_name: snoalb, output_name: SNOWALB} + - {module_name: gfs_phys, field_name: soilm, output_name: SOILM} + - {module_name: gfs_sfc, field_name: SOILT1, output_name: SOILT1} + - {module_name: gfs_sfc, field_name: SOILT2, output_name: SOILT2} + - {module_name: gfs_sfc, field_name: SOILT3, output_name: SOILT3} + - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} +- name: atmos_dt_atmos.zarr + chunks: + time: 4 + times: + frequency: 900 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: area, output_name: area} + - {module_name: dynamics, field_name: us, output_name: UGRDlowest} + - {module_name: dynamics, field_name: u850, output_name: UGRD850} + - {module_name: dynamics, field_name: u500, output_name: UGRD500} + - {module_name: dynamics, field_name: u200, output_name: UGRD200} + - {module_name: dynamics, field_name: u50, output_name: UGRD50} + - {module_name: dynamics, field_name: vs, output_name: VGRDlowest} + - {module_name: dynamics, field_name: v850, output_name: VGRD850} + - {module_name: dynamics, field_name: v500, output_name: VGRD500} + - {module_name: dynamics, field_name: v200, output_name: VGRD200} + - {module_name: dynamics, field_name: v50, output_name: VGRD50} + - {module_name: dynamics, field_name: tm, output_name: TMP500_300} + - {module_name: dynamics, field_name: tb, output_name: TMPlowest} + - {module_name: dynamics, field_name: t850, output_name: TMP850} + - {module_name: dynamics, field_name: t500, output_name: TMP500} + - {module_name: dynamics, field_name: t200, output_name: TMP200} + - {module_name: dynamics, field_name: w850, output_name: w850} + - {module_name: dynamics, field_name: w500, output_name: w500} + - {module_name: dynamics, field_name: w200, output_name: w200} + - {module_name: dynamics, field_name: w50, output_name: w50} + - {module_name: dynamics, field_name: vort850, output_name: VORT850} + - {module_name: dynamics, field_name: vort500, output_name: VORT500} + - {module_name: dynamics, field_name: vort200, output_name: VORT200} + - {module_name: dynamics, field_name: z850, output_name: h850} + - {module_name: dynamics, field_name: z500, output_name: h500} + - {module_name: dynamics, field_name: z200, output_name: h200} + - {module_name: dynamics, field_name: rh1000, output_name: RH1000} + - {module_name: dynamics, field_name: rh925, output_name: RH925} + - {module_name: dynamics, field_name: rh850, output_name: RH850} + - {module_name: dynamics, field_name: rh700, output_name: RH700} + - {module_name: dynamics, field_name: rh500, output_name: RH500} + - {module_name: dynamics, field_name: rh200, output_name: RH200} + - {module_name: dynamics, field_name: q1000, output_name: q1000} + - {module_name: dynamics, field_name: q925, output_name: q925} + - {module_name: dynamics, field_name: q850, output_name: q850} + - {module_name: dynamics, field_name: q700, output_name: q700} + - {module_name: dynamics, field_name: q500, output_name: q500} + - {module_name: dynamics, field_name: q200, output_name: q200} + - {module_name: dynamics, field_name: slp, output_name: PRMSL} + - {module_name: dynamics, field_name: ps, output_name: PRESsfc} + - {module_name: dynamics, field_name: tq, output_name: PWAT} + - {module_name: dynamics, field_name: lw, output_name: VIL} + - {module_name: dynamics, field_name: iw, output_name: iw} + - {module_name: dynamics, field_name: ke, output_name: kinetic_energy} + - {module_name: dynamics, field_name: te, output_name: total_energy} + - {module_name: dynamics, field_name: column_fv_sat_adj_heating, output_name: column_fv_sat_adj_heating} + - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} +- name: atmos_dt_atmos_3d.zarr + chunks: + time: 1 + times: + frequency: 3600 + kind: interval + variables: + - {module_name: dynamics, field_name: grid_lont, output_name: lon} + - {module_name: dynamics, field_name: grid_latt, output_name: lat} + - {module_name: dynamics, field_name: grid_lon, output_name: lonb} + - {module_name: dynamics, field_name: grid_lat, output_name: latb} + - {module_name: dynamics, field_name: fv_sat_adj_qv_dt, output_name: fv_sat_adj_qv_dt} + - {module_name: dynamics, field_name: fv_sat_adj_t_dt, output_name: fv_sat_adj_t_dt} diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-max-random-config.yaml similarity index 69% rename from projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-config.yaml rename to projects/cloud_ml/argo/prognostic-run/prescribed-cloud-max-random-config.yaml index 114300d19c..adf6e3334a 100644 --- a/projects/cloud_ml/argo/nudge-to-fine-run/nudge-to-fine-config.yaml +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-max-random-config.yaml @@ -1,116 +1,94 @@ base_version: v0.7 -field_table: gs://vcm-fv3config/config/field_table/TKE-EDMF/v1.0/field_table prephysics: - - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/C3072-surface-radiative-fluxes-precip.zarr" + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" variables: - - override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface - - override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface - - override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface - - total_precipitation + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" nudging: - restarts_path: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts timescale_hours: air_temperature: 3 specific_humidity: 3 x_wind: 3 y_wind: 3 pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 namelist: atmos_model_nml: fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved coupler_nml: - days: 5 - hours: 0 + days: 0 + hours: 6 minutes: 0 months: 0 seconds: 0 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 gfs_physics_nml: - fhlwr: 1800.0 - fhswr: 1800.0 + fhzero: 0.25 + fhlwr: 900.0 + fhswr: 900.0 + iovr_sw: 1 + iovr_lw: 1 + ccnorm: false override_surface_radiative_fluxes: true - hybedmf: false - satmedmf: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files diagnostics: -- name: state_after_timestep.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval - variables: - - x_wind - - y_wind - - eastward_wind - - northward_wind - - vertical_wind - - air_temperature - - specific_humidity - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - land_sea_mask - - surface_temperature - - surface_geopotential - - surface_diffused_shortwave_albedo - - sensible_heat_flux - - latent_heat_flux - - total_precipitation - - surface_precipitation_rate - - total_soil_moisture - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_surface - - total_sky_downward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - latitude - - longitude - - cloud_water_mixing_ratio - - cloud_ice_mixing_ratio - - rain_mixing_ratio - - snow_mixing_ratio - - graupel_mixing_ratio - - cloud_amount -- name: physics_tendencies.zarr +- name: radiative_fluxes.zarr chunks: - time: 16 + time: 4 times: - frequency: 5400 + frequency: 900 kind: interval variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics -- name: nudging_tendencies.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_tendency_due_to_nudging - - specific_humidity_tendency_due_to_nudging - - x_wind_tendency_due_to_nudging - - y_wind_tendency_due_to_nudging - - pressure_thickness_of_atmospheric_layer_tendency_due_to_nudging + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python - name: diags.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -121,22 +99,10 @@ diagnostics: - total_precipitation_rate - water_vapor_path - physics_precip -- name: reference_state.zarr - chunks: - time: 8 - times: - frequency: 10800 - kind: interval-average - variables: - - air_temperature_reference - - specific_humidity_reference - - x_wind_reference - - y_wind_reference - - pressure_thickness_of_atmospheric_layer_reference fortran_diagnostics: - name: sfc_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -155,14 +121,27 @@ fortran_diagnostics: - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} @@ -183,7 +162,7 @@ fortran_diagnostics: - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} - name: atmos_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -241,9 +220,9 @@ fortran_diagnostics: - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} - name: atmos_dt_atmos_3d.zarr chunks: - time: 16 + time: 1 times: - frequency: 5400 + frequency: 3600 kind: interval variables: - {module_name: dynamics, field_name: grid_lont, output_name: lon} diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/baseline-v3-config.yaml b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-random-config.yaml similarity index 65% rename from projects/cloud_ml/argo/nudge-to-fine-run/baseline-v3-config.yaml rename to projects/cloud_ml/argo/prognostic-run/prescribed-cloud-random-config.yaml index 260af09e73..efcaf3191a 100644 --- a/projects/cloud_ml/argo/nudge-to-fine-run/baseline-v3-config.yaml +++ b/projects/cloud_ml/argo/prognostic-run/prescribed-cloud-random-config.yaml @@ -1,108 +1,108 @@ base_version: v0.7 -field_table: gs://vcm-fv3config/config/field_table/TKE-EDMF/v1.0/field_table prephysics: - - dataset_key: "gs://vcm-ml-intermediate/2022-03-08-C3072-reference-SSTs-at-C48-C384/C48.zarr" + - dataset_key: "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" variables: - - ocean_surface_temperature - reference_initial_time: "20160802.000000" - reference_frequency_seconds: 86400 + override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface + override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface: override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface + override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface: override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface + total_precipitation: total_precipitation initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" + base_url: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timestep: "20200731.003000" +nudging: + restarts_path: gs://vcm-ml-raw-flexible-retention/2023-05-22-PIRE-like-C3072-ccnorm-true-simulation/C3072-to-C48-restarts + timescale_hours: + air_temperature: 3 + specific_humidity: 3 + x_wind: 3 + y_wind: 3 + pressure_thickness_of_atmospheric_layer: 3 +radiation_scheme: + kind: "python" + input_generator: + dataset_key: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr + variables: + cloud_water_mixing_ratio: cloud_water_mixing_ratio + cloud_ice_mixing_ratio: cloud_ice_mixing_ratio + cloud_amount: cloud_amount + reference_initial_time: "20200731.003000" + reference_frequency_seconds: 900 namelist: atmos_model_nml: fhmax: 1024 # max hours of simulation (over all segments) for which fortran physics diags are saved coupler_nml: - days: 1 - hours: 0 + days: 0 + hours: 6 minutes: 0 months: 0 seconds: 0 - restart_secs: 900 + gfdl_cloud_microphysics_nml: + mp_time: 150.0 gfs_physics_nml: + fhzero: 0.25 fhlwr: 900.0 fhswr: 900.0 + iovr_sw: 0 + iovr_lw: 0 + ccnorm: false override_surface_radiative_fluxes: true - hybedmf: false - satmedmf: true + hybedmf: true + satmedmf: false + xkzm_h: 2.0 + fv_core_nml: + layout: + - 2 + - 2 namsfc: + ftsfs: 90 # SSTs should be overwritten by nudging, anyway + fsnol: 0 # set snow covered area to climatology + fsnos: 0 # set snow covered area to climatology faisl: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files faiss: 99999 # Persist land / sea / sea-ice mask from the initial conditions instead of using climatology forcing files fsicl: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files fsics: 99999 # Persist the sea ice fraction from the initial conditiond instead of using climatology forcing files diagnostics: -- name: state_after_timestep.zarr +- name: radiative_fluxes.zarr chunks: - time: 24 + time: 4 times: frequency: 900 kind: interval variables: - - x_wind - - y_wind - - eastward_wind - - northward_wind - - vertical_wind - - air_temperature - - specific_humidity - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - land_sea_mask - - surface_temperature - - surface_geopotential - - surface_diffused_shortwave_albedo - - sensible_heat_flux - - latent_heat_flux - - total_precipitation - - surface_precipitation_rate - - total_soil_moisture - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_surface - - total_sky_downward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - latitude - - longitude - - cloud_water_mixing_ratio - - cloud_ice_mixing_ratio - - rain_mixing_ratio - - snow_mixing_ratio - - graupel_mixing_ratio - - cloud_amount - - orographic_variables -- name: physics_tendencies.zarr - chunks: - time: 24 - times: - frequency: 900 - kind: interval - variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics + - clear_sky_downward_longwave_flux_at_surface_python + - clear_sky_downward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_surface_python + - clear_sky_upward_shortwave_flux_at_surface_python + - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python + - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_downward_longwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_surface_python + - total_sky_upward_longwave_flux_at_surface_python + - total_sky_upward_shortwave_flux_at_surface_python + - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_upward_longwave_flux_at_top_of_atmosphere_python + - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python + - total_sky_longwave_heating_rate_python + - clear_sky_longwave_heating_rate_python + - total_sky_shortwave_heating_rate_python + - clear_sky_shortwave_heating_rate_python - name: diags.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval variables: + - net_moistening_due_to_nudging + - column_heating_due_to_nudging + - net_mass_tendency_due_to_nudging - total_precipitation_rate - water_vapor_path - physics_precip fortran_diagnostics: - name: sfc_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -120,15 +120,28 @@ fortran_diagnostics: - {module_name: gfs_phys, field_name: totsnwb_ave, output_name: SNOWsfc} - {module_name: gfs_phys, field_name: totgrpb_ave, output_name: GRAUPELsfc} - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} -# - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRF_from_rrtmg, output_name: DSWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DSWRFI, output_name: DSWRFIsfc} + - {module_name: gfs_phys, field_name: csdsf_ave, output_name: DCSWRFsfc} - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} -# - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRF_from_rrtmg, output_name: USWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: USWRFI, output_name: USWRFIsfc} + - {module_name: gfs_phys, field_name: csusf_ave, output_name: UCSWRFsfc} - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} + - {module_name: gfs_phys, field_name: sw_dnfxc, output_name: DSWRFItoa} - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} + - {module_name: gfs_phys, field_name: sw_upfxc, output_name: USWRFItoa} + - {module_name: gfs_phys, field_name: csusf_avetoa, output_name: UCSWRFtoa} - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} + - {module_name: gfs_phys, field_name: lw_upfxc, output_name: ULWRFItoa} + - {module_name: gfs_phys, field_name: csulf_avetoa, output_name: UCLWRFtoa} - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} + - {module_name: gfs_phys, field_name: ULWRFI, output_name: ULWRFIsfc} + - {module_name: gfs_phys, field_name: csulf_ave, output_name: UCLWRFsfc} - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} -# - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRF_from_rrtmg, output_name: DLWRFsfc_from_RRTMG} + - {module_name: gfs_phys, field_name: DLWRFI, output_name: DLWRFIsfc} + - {module_name: gfs_phys, field_name: csdlf_ave, output_name: DCLWRFsfc} - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} - {module_name: gfs_phys, field_name: hpbl, output_name: HPBLsfc} @@ -149,7 +162,7 @@ fortran_diagnostics: - {module_name: gfs_sfc, field_name: SOILT4, output_name: SOILT4} - name: atmos_dt_atmos.zarr chunks: - time: 96 + time: 4 times: frequency: 900 kind: interval @@ -207,9 +220,9 @@ fortran_diagnostics: - {module_name: dynamics, field_name: column_fv_sat_adj_moistening, output_name: column_fv_sat_adj_moistening} - name: atmos_dt_atmos_3d.zarr chunks: - time: 24 + time: 1 times: - frequency: 900 + frequency: 3600 kind: interval variables: - {module_name: dynamics, field_name: grid_lont, output_name: lon} diff --git a/projects/cloud_ml/argo/nudge-to-fine-run/run.sh b/projects/cloud_ml/argo/prognostic-run/run.sh similarity index 77% rename from projects/cloud_ml/argo/nudge-to-fine-run/run.sh rename to projects/cloud_ml/argo/prognostic-run/run.sh index 3563e285c2..744d67b610 100755 --- a/projects/cloud_ml/argo/nudge-to-fine-run/run.sh +++ b/projects/cloud_ml/argo/prognostic-run/run.sh @@ -6,7 +6,7 @@ CONFIG=$1 PROJECT=cloud-ml -EXPERIMENT="cloud-ml-training-data" +EXPERIMENT="cloud-ml-prognostic-run" TRIAL="trial-0" TAG=${EXPERIMENT}-${CONFIG} NAME="${TAG}-$(openssl rand --hex 2)" @@ -15,7 +15,8 @@ argo submit --from workflowtemplate/prognostic-run \ -p project=${PROJECT} \ -p tag=${TAG} \ -p config="$(< ${CONFIG}-config.yaml)" \ - -p segment-count="1" \ - -p memory="15Gi" \ + -p segment-count="39" \ + -p cpu="24" \ + -p memory="25Gi" \ --name "${NAME}" \ --labels "project=${PROJECT},experiment=${EXPERIMENT},trial=${TRIAL}" \ No newline at end of file diff --git a/projects/cloud_ml/argo/prognostic-run/run_seed_squash_threshold.sh b/projects/cloud_ml/argo/prognostic-run/run_seed_squash_threshold.sh new file mode 100755 index 0000000000..c5671ac11b --- /dev/null +++ b/projects/cloud_ml/argo/prognostic-run/run_seed_squash_threshold.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +CONFIG=$1 +N_SEEDS=$2 +SEGMENT_COUNT=$3 + +PROJECT=cloud-ml +EXPERIMENT="cloud-ml-prog-run" +TRIAL="trial-0" + +TMP_CONFIG="/tmp/tmp_config.yaml" + +SQUASH_FILE="../../scripts/upload-squashed-models/squash_thresholds.yaml" +readarray -t SQUASH_THRESHOLDS < <(yq '.squash_thresholds[]' ${SQUASH_FILE}) + + +for ((SEED=0;SEED<$N_SEEDS;SEED++)) +do + for SQUASH_THRESHOLD in "${SQUASH_THRESHOLDS[@]}" + do + TAG="${EXPERIMENT}-${CONFIG}-seed-${SEED}-${SQUASH_THRESHOLD}" + NAME="${TAG}-$(openssl rand --hex 2)" + + echo "Preparing seed ${SEED} and squash threshold ${SQUASH_THRESHOLD}." + cp "${CONFIG}-config.yaml" $TMP_CONFIG + yq -yi --arg SEED "${SEED}" '.radiation_scheme.input_generator.model[] |= (sub("seed-0"; "seed-\($SEED)"))' ${TMP_CONFIG} + yq -yi --arg SQUASH_THRESHOLD "${SQUASH_THRESHOLD}" \ + '.radiation_scheme.input_generator.model[] |= (sub("squash-threshold-0"; "squash-threshold-\($SQUASH_THRESHOLD | tonumber)"))' \ + ${TMP_CONFIG} + + argo submit --from workflowtemplate/prognostic-run \ + -p project=${PROJECT} \ + -p tag=${TAG} \ + -p config="$(< ${TMP_CONFIG})" \ + -p segment-count="${SEGMENT_COUNT}" \ + -p cpu="24" \ + -p memory="25Gi" \ + --name "${NAME}" \ + --labels "project=${PROJECT},experiment=${EXPERIMENT},trial=${TRIAL}" + + rm $TMP_CONFIG + done +done \ No newline at end of file diff --git a/projects/cloud_ml/argo/training/dense-config.yaml b/projects/cloud_ml/argo/training/dense-config.yaml new file mode 100644 index 0000000000..c8e348856a --- /dev/null +++ b/projects/cloud_ml/argo/training/dense-config.yaml @@ -0,0 +1,52 @@ +random_seed: 0 +model_type: dense +hyperparameters: + loss: + loss_type: mse + scaling: standard + dense_network: + depth: 4 + width: 169 + kernel_regularizer: + name: l2 + kwargs: + l: 0.0005 + training_loop: + batch_size: 512 + epochs: 20 + in_memory: false + shuffle_buffer_size: 2_184_192 + output_limit_config: + limits: + cloud_amount: {min: 0.0, max: 1.0} + cloud_ice_mixing_ratio: {min: 0.0} + cloud_water_mixing_ratio: {min: 0.0} + optimizer_config: + name: Adam + learning_rate_schedule: + name: ExponentialDecay + kwargs: + initial_learning_rate: 0.001 + decay_steps: 100_000 + decay_rate: 0.96 + staircase: true + callbacks: + - kwargs: {} + name: WandbCallback + - kwargs: + monitor: val_loss + patience: 10 + name: EarlyStopping + - name: EpochModelCheckpoint + kwargs: + filepath: gs://vcm-ml-experiments/cloud-ml/2023-06-07/trained_models_by_epoch/dense-seed-0/{epoch} + input_variables: + - pressure + - relative_humidity + - air_temperature + - cloud_ice_mixing_ratio_coarse + output_variables: + - cloud_ice_mixing_ratio + - cloud_water_mixing_ratio + - cloud_amount + predict_columns: false \ No newline at end of file diff --git a/projects/cloud_ml/argo/training/run_random_seed.sh b/projects/cloud_ml/argo/training/run_random_seed.sh new file mode 100755 index 0000000000..95ede66f4f --- /dev/null +++ b/projects/cloud_ml/argo/training/run_random_seed.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e +set -x + +TRAINING_CONFIG=$1 +N_SEEDS=$2 + +BUCKET="vcm-ml-experiments" +PROJECT="cloud-ml" +EXPERIMENT="train-cloud-ml-${TRAINING_CONFIG}" +TAG="${EXPERIMENT}" +WANDB_GROUP="${EXPERIMENT}" +WANDB_PROJECT="radiation-cloud-ml" + +TMP_CONFIG="tmp-config.yaml" + +for ((SEED=0;SEED<$N_SEEDS;SEED++)) +do + NAME="${TAG}-$(openssl rand --hex 2)" + + echo "Preparing seed ${SEED}..." + cp "${TRAINING_CONFIG}-config.yaml" $TMP_CONFIG + sed -i "s/random_seed: 0/random_seed: $SEED/" $TMP_CONFIG + sed -i "s/seed-0/seed-$SEED/" $TMP_CONFIG + + + argo submit --from workflowtemplate/training \ + -p training_config="$( yq . ${TMP_CONFIG})" \ + -p training_data_config="$( yq . train.yaml )" \ + -p validation_data_config="$( yq . validation.yaml )" \ + -p output="gs://${BUCKET}/${PROJECT}/$(date +%F)/${TAG}/trained_models/${TRAINING_CONFIG}-seed-${SEED}" \ + -p memory="25Gi" \ + -p cpu="4000m" \ + -p flags="--cache.local_download_path train-data-download-dir" \ + -p wandb-project="${WANDB_PROJECT}" \ + -p wandb-group="${WANDB_GROUP}" \ + --name "${NAME}" \ + --labels "project=${PROJECT},experiment=${EXPERIMENT},trial=trial-0" + + rm $TMP_CONFIG +done \ No newline at end of file diff --git a/projects/cloud_ml/argo/training/train.yaml b/projects/cloud_ml/argo/training/train.yaml new file mode 100644 index 0000000000..609b98604a --- /dev/null +++ b/projects/cloud_ml/argo/training/train.yaml @@ -0,0 +1,175 @@ +mapper_config: + function: open_zarr + kwargs: + data_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr +timesteps: +- '20200731.003000' +- '20200731.013000' +- '20200731.023000' +- '20200731.033000' +- '20200731.043000' +- '20200731.053000' +- '20200731.063000' +- '20200731.073000' +- '20200731.083000' +- '20200731.093000' +- '20200731.103000' +- '20200731.113000' +- '20200731.123000' +- '20200731.133000' +- '20200731.143000' +- '20200731.153000' +- '20200731.163000' +- '20200731.173000' +- '20200731.183000' +- '20200731.193000' +- '20200731.203000' +- '20200731.213000' +- '20200731.223000' +- '20200731.233000' +- '20200801.003000' +- '20200801.013000' +- '20200801.023000' +- '20200801.033000' +- '20200801.043000' +- '20200801.053000' +- '20200801.063000' +- '20200801.073000' +- '20200801.083000' +- '20200801.093000' +- '20200801.103000' +- '20200801.113000' +- '20200801.123000' +- '20200801.133000' +- '20200801.143000' +- '20200801.153000' +- '20200801.163000' +- '20200801.173000' +- '20200801.183000' +- '20200801.193000' +- '20200801.203000' +- '20200801.213000' +- '20200801.223000' +- '20200801.233000' +- '20200802.003000' +- '20200802.013000' +- '20200802.023000' +- '20200802.033000' +- '20200802.043000' +- '20200802.053000' +- '20200802.063000' +- '20200802.073000' +- '20200802.083000' +- '20200802.093000' +- '20200802.103000' +- '20200802.113000' +- '20200802.123000' +- '20200802.133000' +- '20200802.143000' +- '20200802.153000' +- '20200802.163000' +- '20200802.173000' +- '20200802.183000' +- '20200802.193000' +- '20200802.203000' +- '20200802.213000' +- '20200802.223000' +- '20200802.233000' +- '20200803.003000' +- '20200803.013000' +- '20200803.023000' +- '20200803.033000' +- '20200803.043000' +- '20200803.053000' +- '20200803.063000' +- '20200803.073000' +- '20200803.083000' +- '20200803.093000' +- '20200803.103000' +- '20200803.113000' +- '20200803.123000' +- '20200803.133000' +- '20200803.143000' +- '20200803.153000' +- '20200803.163000' +- '20200803.173000' +- '20200803.183000' +- '20200803.193000' +- '20200803.203000' +- '20200803.213000' +- '20200803.223000' +- '20200803.233000' +- '20200804.003000' +- '20200804.013000' +- '20200804.023000' +- '20200804.033000' +- '20200804.043000' +- '20200804.053000' +- '20200804.063000' +- '20200804.073000' +- '20200804.083000' +- '20200804.093000' +- '20200804.103000' +- '20200804.113000' +- '20200804.123000' +- '20200804.133000' +- '20200804.143000' +- '20200804.153000' +- '20200804.163000' +- '20200804.173000' +- '20200804.183000' +- '20200804.193000' +- '20200804.203000' +- '20200804.213000' +- '20200804.223000' +- '20200804.233000' +- '20200805.003000' +- '20200805.013000' +- '20200805.023000' +- '20200805.033000' +- '20200805.043000' +- '20200805.053000' +- '20200805.063000' +- '20200805.073000' +- '20200805.083000' +- '20200805.093000' +- '20200805.103000' +- '20200805.113000' +- '20200805.123000' +- '20200805.133000' +- '20200805.143000' +- '20200805.153000' +- '20200805.163000' +- '20200805.173000' +- '20200805.183000' +- '20200805.193000' +- '20200805.203000' +- '20200805.213000' +- '20200805.223000' +- '20200805.233000' +- '20200806.003000' +- '20200806.013000' +- '20200806.023000' +- '20200806.033000' +- '20200806.043000' +- '20200806.053000' +- '20200806.063000' +- '20200806.073000' +- '20200806.083000' +- '20200806.093000' +- '20200806.103000' +- '20200806.113000' +- '20200806.123000' +- '20200806.133000' +- '20200806.143000' +- '20200806.153000' +- '20200806.163000' +- '20200806.173000' +- '20200806.183000' +- '20200806.193000' +- '20200806.203000' +- '20200806.213000' +- '20200806.223000' +- '20200806.233000' +timesteps_per_batch: 1 +unstacked_dims: [] \ No newline at end of file diff --git a/projects/cloud_ml/argo/training/validation.yaml b/projects/cloud_ml/argo/training/validation.yaml new file mode 100644 index 0000000000..1c82d0f0df --- /dev/null +++ b/projects/cloud_ml/argo/training/validation.yaml @@ -0,0 +1,78 @@ +mapper_config: + function: open_zarr + kwargs: + data_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr +timesteps: +- '20200807.003000' +- '20200807.013000' +- '20200807.023000' +- '20200807.033000' +- '20200807.043000' +- '20200807.053000' +- '20200807.063000' +- '20200807.073000' +- '20200807.083000' +- '20200807.093000' +- '20200807.103000' +- '20200807.113000' +- '20200807.123000' +- '20200807.133000' +- '20200807.143000' +- '20200807.153000' +- '20200807.163000' +- '20200807.173000' +- '20200807.183000' +- '20200807.193000' +- '20200807.203000' +- '20200807.213000' +- '20200807.223000' +- '20200807.233000' +- '20200808.003000' +- '20200808.013000' +- '20200808.023000' +- '20200808.033000' +- '20200808.043000' +- '20200808.053000' +- '20200808.063000' +- '20200808.073000' +- '20200808.083000' +- '20200808.093000' +- '20200808.103000' +- '20200808.113000' +- '20200808.123000' +- '20200808.133000' +- '20200808.143000' +- '20200808.153000' +- '20200808.163000' +- '20200808.173000' +- '20200808.183000' +- '20200808.193000' +- '20200808.203000' +- '20200808.213000' +- '20200808.223000' +- '20200808.233000' +- '20200809.003000' +- '20200809.013000' +- '20200809.023000' +- '20200809.033000' +- '20200809.043000' +- '20200809.053000' +- '20200809.063000' +- '20200809.073000' +- '20200809.083000' +- '20200809.093000' +- '20200809.103000' +- '20200809.113000' +- '20200809.123000' +- '20200809.133000' +- '20200809.143000' +- '20200809.153000' +- '20200809.163000' +- '20200809.173000' +- '20200809.183000' +- '20200809.193000' +- '20200809.203000' +- '20200809.213000' +- '20200809.223000' +timesteps_per_batch: 1 +unstacked_dims: [] \ No newline at end of file diff --git a/projects/cloud_ml/prognostic_run/radiation_piggyback.yaml b/projects/cloud_ml/prognostic_run/radiation_piggyback.yaml deleted file mode 100644 index 0ede575f0f..0000000000 --- a/projects/cloud_ml/prognostic_run/radiation_piggyback.yaml +++ /dev/null @@ -1,142 +0,0 @@ -base_version: v0.7 -initial_conditions: - base_url: gs://vcm-ml-experiments/2020-06-02-fine-res/coarsen_restarts - timestep: "20160805.000000" -radiation_scheme: - kind: "python" -# input_model: # uncomment this to make predictions with an ML model -# model: -# - gs://vcm-ml-experiments/cloud-ml/2022-08-29/example-cloud-model-trial-0/trained_models/cloud_model -namelist: - coupler_nml: - hours: 0 - minutes: 30 - gfdl_cloud_microphysics_nml: - fast_sat_adj: false - gfs_physics_nml: - fhlwr: 900.0 - fhswr: 900.0 - hybedmf: true - satmedmf: false -fortran_diagnostics: - - name: atmos_dt_atmos.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - {module_name: dynamics, field_name: grid_lont, output_name: lon} - - {module_name: dynamics, field_name: grid_latt, output_name: lat} - - {module_name: dynamics, field_name: grid_lon, output_name: lonb} - - {module_name: dynamics, field_name: grid_lat, output_name: latb} - - {module_name: dynamics, field_name: area, output_name: area} - - {module_name: dynamics, field_name: tb, output_name: TMPlowest} - - {module_name: dynamics, field_name: t850, output_name: TMP850} - - {module_name: dynamics, field_name: t500, output_name: TMP500} - - {module_name: dynamics, field_name: t200, output_name: TMP200} - - {module_name: dynamics, field_name: w500, output_name: w500} - - {module_name: dynamics, field_name: vort500, output_name: VORT500} - - {module_name: dynamics, field_name: z500, output_name: h500} - - {module_name: dynamics, field_name: rh850, output_name: RH850} - - {module_name: dynamics, field_name: q500, output_name: q500} - - {module_name: dynamics, field_name: ps, output_name: PRESsfc} - - {module_name: dynamics, field_name: tq, output_name: PWAT} - - {module_name: dynamics, field_name: lw, output_name: VIL} - - name: sfc_dt_atmos.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - {module_name: dynamics, field_name: grid_lont, output_name: lon} - - {module_name: dynamics, field_name: grid_latt, output_name: lat} - - {module_name: dynamics, field_name: grid_lon, output_name: lonb} - - {module_name: dynamics, field_name: grid_lat, output_name: latb} - - {module_name: dynamics, field_name: area, output_name: area} - - {module_name: gfs_phys, field_name: dusfci, output_name: uflx} - - {module_name: gfs_phys, field_name: dvsfci, output_name: vflx} - - {module_name: gfs_phys, field_name: cnvprcpb_ave, output_name: CPRATsfc} - - {module_name: gfs_phys, field_name: totprcpb_ave, output_name: PRATEsfc} - - {module_name: gfs_phys, field_name: DSWRF, output_name: DSWRFsfc} - - {module_name: gfs_phys, field_name: USWRF, output_name: USWRFsfc} - - {module_name: gfs_phys, field_name: DSWRFtoa, output_name: DSWRFtoa} - - {module_name: gfs_phys, field_name: USWRFtoa, output_name: USWRFtoa} - - {module_name: gfs_phys, field_name: ULWRFtoa, output_name: ULWRFtoa} - - {module_name: gfs_phys, field_name: ULWRF, output_name: ULWRFsfc} - - {module_name: gfs_phys, field_name: DLWRF, output_name: DLWRFsfc} - - {module_name: gfs_phys, field_name: lhtfl_ave, output_name: LHTFLsfc} - - {module_name: gfs_phys, field_name: shtfl_ave, output_name: SHTFLsfc} -diagnostics: - - name: diags.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - total_precipitation_rate - - water_vapor_path - - physics_precip - - name: diags_3d.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - tendency_of_air_temperature_due_to_fv3_physics - - tendency_of_specific_humidity_due_to_fv3_physics - - tendency_of_eastward_wind_due_to_fv3_physics - - tendency_of_northward_wind_due_to_fv3_physics - - name: state_after_timestep.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - eastward_wind - - northward_wind - - vertical_wind - - air_temperature - - specific_humidity - - pressure_thickness_of_atmospheric_layer - - vertical_thickness_of_atmospheric_layer - - land_sea_mask - - surface_temperature - - surface_geopotential - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_downward_shortwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_downward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_surface - - total_sky_upward_longwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - name: radiative_fluxes.zarr - chunks: - time: 2 - times: - frequency: 900 - kind: interval - variables: - - clear_sky_downward_longwave_flux_at_surface_python - - clear_sky_downward_shortwave_flux_at_surface_python - - clear_sky_upward_longwave_flux_at_surface_python - - clear_sky_upward_shortwave_flux_at_surface_python - - clear_sky_upward_longwave_flux_at_top_of_atmosphere_python - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere_python - - total_sky_downward_longwave_flux_at_surface_python - - total_sky_downward_shortwave_flux_at_surface_python - - total_sky_upward_longwave_flux_at_surface_python - - total_sky_upward_shortwave_flux_at_surface_python - - total_sky_downward_shortwave_flux_at_top_of_atmosphere_python - - total_sky_upward_longwave_flux_at_top_of_atmosphere_python - - total_sky_upward_shortwave_flux_at_top_of_atmosphere_python \ No newline at end of file diff --git a/projects/cloud_ml/scripts/fine-coarse-3d-merged-dataset-v4.py b/projects/cloud_ml/scripts/fine-coarse-3d-merged-dataset-v4.py deleted file mode 100644 index 5f695d7771..0000000000 --- a/projects/cloud_ml/scripts/fine-coarse-3d-merged-dataset-v4.py +++ /dev/null @@ -1,152 +0,0 @@ -import intake -import xarray as xr -import os -from dask.diagnostics import ProgressBar -import fsspec - -from vcm.catalog import catalog as CATALOG -from vcm.fv3.metadata import standardize_fv3_diagnostics -from vcm import convert_timestamps -from vcm.cubedsphere import center_and_rotate_xy_winds -from vcm.safe import get_variables - - -WIND_ROTATION_MATRIX = CATALOG["wind_rotation/c48"].to_dask() -GRID = CATALOG["grid/c48"].to_dask() -COARSE_NUDGED_PATH = "gs://vcm-ml-experiments/cloud-ml/2022-08-23/cloud-ml-training-data-nudge-to-fine-v4/fv3gfs_run" # noqa: E501 -FINE_RESTARTS_KEYS = [ - "40day_c48_restarts_as_zarr_may2020", - "40day_c48_gfsphysics_15min_may2020", -] -FINE_TO_COARSE_RENAME = { - "T": "air_temperature", - "sphum": "specific_humidity", - "eastward_wind": "eastward_wind", - "northward_wind": "northward_wind", - "DZ": "vertical_thickness_of_atmospheric_layer", - "delp": "pressure_thickness_of_atmospheric_layer", - "liq_wat": "cloud_water_mixing_ratio", - "ice_wat": "cloud_ice_mixing_ratio", - "rainwat": "rain_mixing_ratio", - "snowwat": "snow_mixing_ratio", - "graupel": "graupel_mixing_ratio", - "cld_amt": "cloud_amount", - "DCLWRFsfc": "clear_sky_downward_longwave_flux_at_surface", - "DCSWRFsfc": "clear_sky_downward_shortwave_flux_at_surface", - "DLWRFsfc": "total_sky_downward_longwave_flux_at_surface", - "DSWRFsfc": "total_sky_downward_shortwave_flux_at_surface", - "DSWRFtoa": "total_sky_downward_shortwave_flux_at_top_of_atmosphere", - "UCLWRFsfc": "clear_sky_upward_longwave_flux_at_surface", - "UCLWRFtoa": "clear_sky_upward_longwave_flux_at_top_of_atmosphere", - "UCSWRFsfc": "clear_sky_upward_shortwave_flux_at_surface", - "UCSWRFtoa": "clear_sky_upward_shortwave_flux_at_top_of_atmosphere", - "ULWRFsfc": "total_sky_upward_longwave_flux_at_surface", - "ULWRFtoa": "total_sky_upward_longwave_flux_at_top_of_atmosphere", - "USWRFsfc": "total_sky_upward_shortwave_flux_at_surface", - "USWRFtoa": "total_sky_upward_shortwave_flux_at_top_of_atmosphere", -} -RENAME_DIMS = {"pfull": "z"} -COORD_VARS = ["x", "y", "z", "tile"] -OUTPUT_CHUNKS = {"tile": 6} -OUTPUT_PATH = "gs://vcm-ml-experiments/cloud-ml/2022-08-23/fine-coarse-3d-fields.zarr" - - -def rotate_winds(ds): - eastward_wind, northward_wind = center_and_rotate_xy_winds( - WIND_ROTATION_MATRIX, ds.u, ds.v - ) - ds["eastward_wind"] = eastward_wind - ds["northward_wind"] = northward_wind - return ds.drop_vars(["u", "v"]) - - -def get_fine_ds(): - datasets = [] - for key in FINE_RESTARTS_KEYS: - dataset = CATALOG[key].to_dask() - if isinstance(dataset.time[0].item(), str): - dataset = dataset.assign_coords({"time": convert_timestamps(dataset.time)}) - dataset = standardize_fv3_diagnostics(dataset) - if "pfull" in dataset.dims: - dataset = dataset.rename(RENAME_DIMS) - datasets.append(dataset) - ds = xr.merge(datasets) - rotate_winds(ds) - ds_3d = xr.Dataset() - for restart_name, python_name in FINE_TO_COARSE_RENAME.items(): - ds_3d[python_name] = ds[restart_name] - return ds_3d.drop_vars(COORD_VARS) - - -def get_coarse_ds(): - full_path = os.path.join(COARSE_NUDGED_PATH, "state_after_timestep.zarr") - ds = intake.open_zarr(full_path, consolidated=True).to_dask() - ds_3d = xr.Dataset() - for var in FINE_TO_COARSE_RENAME.values(): - ds_3d[var] = ds[var] - return ds_3d - - -def merge_coarse_fine(coarse, fine): - # subset times and variables to intersection - common_times = xr.DataArray( - data=sorted(list(set(coarse.time.values).intersection(set(fine.time.values)))), - dims=["time"], - ) - coarse, fine = coarse.sel(time=common_times), fine.sel(time=common_times) - common_vars = set(coarse.data_vars).intersection(fine.data_vars) - coarse, fine = get_variables(coarse, common_vars), get_variables(fine, common_vars) - - # add 'res' dimension and concatenate - merged = xr.concat( - [coarse.expand_dims({"res": ["coarse"]}), fine.expand_dims({"res": ["fine"]})], - dim="res", - ) - # add grid variables back - return xr.merge([merged, GRID]) - - -def main(): - """ - Let's make a merged dataset of fine and coarse 3D (really, time-tile-x-y-z) - variables, including the water tracer species. Specifically: - - - air temperature - - specific humidity - - northward wind - - eastward wind - - layer pressure thickness - - layer height thickness - - cloud water - - cloud ice - - graupel water - - rain water - - snow water - - cloud fraction - - clear_sky_downward_longwave_flux_at_surface - - clear_sky_downward_shortwave_flux_at_surface - - total_sky_downward_longwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_surface - - total_sky_downward_shortwave_flux_at_top_of_atmosphere - - clear_sky_upward_longwave_flux_at_surface - - clear_sky_upward_longwave_flux_at_top_of_atmosphere - - clear_sky_upward_shortwave_flux_at_surface - - clear_sky_upward_shortwave_flux_at_top_of_atmosphere - - total_sky_upward_longwave_flux_at_surface - - total_sky_upward_longwave_flux_at_top_of_atmosphere - - total_sky_upward_shortwave_flux_at_surface - - total_sky_upward_shortwave_flux_at_top_of_atmosphere - - """ - - fine = get_fine_ds() - coarse = get_coarse_ds() - merged = merge_coarse_fine(coarse, fine) - - merged = merged.unify_chunks().chunk(OUTPUT_CHUNKS) - with ProgressBar(): - merged.to_zarr(fsspec.get_mapper(OUTPUT_PATH), consolidated=True) - - -if __name__ == "__main__": - main() diff --git a/projects/cloud_ml/scripts/fine-coarse-3d-merged.py b/projects/cloud_ml/scripts/fine-coarse-3d-merged.py deleted file mode 100644 index 795924e456..0000000000 --- a/projects/cloud_ml/scripts/fine-coarse-3d-merged.py +++ /dev/null @@ -1,116 +0,0 @@ -import intake -import xarray as xr -import os -from dask.diagnostics import ProgressBar -import fsspec - -from vcm.catalog import catalog as CATALOG -from vcm.fv3.metadata import standardize_fv3_diagnostics -from vcm import convert_timestamps -from vcm.cubedsphere import center_and_rotate_xy_winds -from vcm.safe import get_variables - - -WIND_ROTATION_MATRIX = CATALOG["wind_rotation/c48"].to_dask() -GRID = CATALOG["grid/c48"].to_dask() -COARSE_NUDGED_PATH = "gs://vcm-ml-experiments/cloud-ml/2022-06-24/cloud-ml-training-data-trial-0/fv3gfs_run" # noqa: E501 -FINE_RESTARTS_KEY = "40day_c48_restarts_as_zarr_may2020" -FINE_TO_COARSE_RENAME = { - "T": "air_temperature", - "sphum": "specific_humidity", - "eastward_wind": "eastward_wind", - "northward_wind": "northward_wind", - "DZ": "vertical_thickness_of_atmospheric_layer", - "delp": "pressure_thickness_of_atmospheric_layer", - "liq_wat": "cloud_water_mixing_ratio", - "ice_wat": "cloud_ice_mixing_ratio", - "rainwat": "rain_mixing_ratio", - "snowwat": "snow_mixing_ratio", - "graupel": "graupel_mixing_ratio", - "cld_amt": "cloud_amount", -} -RENAME_DIMS = {"pfull": "z"} -COORD_VARS = ["x", "y", "z", "tile"] -OUTPUT_CHUNKS = {"tile": 6} -OUTPUT_PATH = "gs://vcm-ml-experiments/cloud-ml/2022-06-24/fine-coarse-3d-fields.zarr" - - -def rotate_winds(ds): - eastward_wind, northward_wind = center_and_rotate_xy_winds( - WIND_ROTATION_MATRIX, ds.u, ds.v - ) - ds["eastward_wind"] = eastward_wind - ds["northward_wind"] = northward_wind - return ds.drop_vars(["u", "v"]) - - -def get_fine_ds(): - ds = CATALOG[FINE_RESTARTS_KEY].to_dask() - ds = ds.assign_coords({"time": convert_timestamps(ds.time)}) - ds = standardize_fv3_diagnostics(ds).rename(RENAME_DIMS) - rotate_winds(ds) - ds_3d = xr.Dataset() - for restart_name, python_name in FINE_TO_COARSE_RENAME.items(): - ds_3d[python_name] = ds[restart_name] - return ds_3d.drop_vars(COORD_VARS) - - -def get_coarse_ds(): - full_path = os.path.join(COARSE_NUDGED_PATH, "state_after_timestep.zarr") - ds = intake.open_zarr(full_path, consolidated=True).to_dask() - ds_3d = xr.Dataset() - for var in FINE_TO_COARSE_RENAME.values(): - ds_3d[var] = ds[var] - return ds_3d - - -def merge_coarse_fine(coarse, fine): - # subset times and variables to intersection - common_times = xr.DataArray( - data=sorted(list(set(coarse.time.values).intersection(set(fine.time.values)))), - dims=["time"], - ) - coarse, fine = coarse.sel(time=common_times), fine.sel(time=common_times) - common_vars = set(coarse.data_vars).intersection(fine.data_vars) - coarse, fine = get_variables(coarse, common_vars), get_variables(fine, common_vars) - - # add 'res' dimension and concatenate - merged = xr.concat( - [coarse.expand_dims({"res": ["coarse"]}), fine.expand_dims({"res": ["fine"]})], - dim="res", - ) - # add grid variables back - return xr.merge([merged, GRID]) - - -def main(): - """ - Let's make a merged dataset of fine and coarse 3D (really, time-tile-x-y-z) - variables, including the water tracer species. Specifically: - - - air temperature - - specific humidity - - northward wind - - eastward wind - - layer pressure thickness - - layer height thickness - - cloud water - - cloud ice - - graupel water - - rain water - - snow water - - cloud fraction - - """ - - fine = get_fine_ds() - coarse = get_coarse_ds() - merged = merge_coarse_fine(coarse, fine) - - merged = merged.unify_chunks().chunk(OUTPUT_CHUNKS) - with ProgressBar(): - merged.to_zarr(fsspec.get_mapper(OUTPUT_PATH), consolidated=True) - - -if __name__ == "__main__": - main() diff --git a/projects/cloud_ml/scripts/offline_predictions_config.yaml b/projects/cloud_ml/scripts/offline_predictions_config.yaml new file mode 100644 index 0000000000..9268d6dc59 --- /dev/null +++ b/projects/cloud_ml/scripts/offline_predictions_config.yaml @@ -0,0 +1,18 @@ +input_data_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr +model_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/train-cloud-ml-dense/squashed-models/dense-seed-3/squash-threshold-0 +output_path: gs://vcm-ml-experiments/cloud-ml/2023-06-12/predicted-fine-cloud-fields-seed-3-squash-threshold-0.zarr +start_time: + - 2020 + - 8 + - 7 + - 0 + - 0 + - 0 +end_time: + - 2020 + - 8 + - 9 + - 23 + - 0 + - 0 +time_stride: 4 \ No newline at end of file diff --git a/projects/cloud_ml/scripts/offline_predictions_squashed_config.yaml b/projects/cloud_ml/scripts/offline_predictions_squashed_config.yaml new file mode 100644 index 0000000000..b60b7ca5c8 --- /dev/null +++ b/projects/cloud_ml/scripts/offline_predictions_squashed_config.yaml @@ -0,0 +1,18 @@ +input_data_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr +model_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/train-cloud-ml-dense/squashed-models/dense-seed-3/squash-threshold-0.08 +output_path: gs://vcm-ml-experiments/cloud-ml/2023-06-12/predicted-fine-cloud-fields-seed-3-squash-threshold-0.08.zarr +start_time: + - 2020 + - 8 + - 7 + - 0 + - 0 + - 0 +end_time: + - 2020 + - 8 + - 9 + - 23 + - 0 + - 0 +time_stride: 4 \ No newline at end of file diff --git a/projects/cloud_ml/scripts/offline_predictions_zarr.py b/projects/cloud_ml/scripts/offline_predictions_zarr.py new file mode 100644 index 0000000000..7f7c92c14e --- /dev/null +++ b/projects/cloud_ml/scripts/offline_predictions_zarr.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import cftime +import sys +import yaml +import xarray as xr +import fv3fit +from vcm import DerivedMapping + +INPUT_VARIABLES = [ + "pressure", + "relative_humidity", + "air_temperature", + "cloud_ice_mixing_ratio_coarse", +] +OUTPUT_VARIABLES = [ + "cloud_ice_mixing_ratio", + "cloud_water_mixing_ratio", + "cloud_amount", +] + + +def get_config(path): + if path is not None: + with open(path, "r") as f: + return yaml.safe_load(f) + else: + return {} + + +def main(config): + model_path = config["model_path"] + output_path = config["output_path"] + model = fv3fit.load(model_path) + coarse_ds = xr.open_zarr(config["input_data_path"]) + inputs = DerivedMapping(coarse_ds).dataset(model.input_variables) + start_time = cftime.DatetimeJulian(*config["start_time"]) + end_time = cftime.DatetimeJulian(*config["end_time"]) + time_stride = config.get("time_stride", None) + inputs = inputs.sel(time=slice(start_time, end_time, time_stride)) + input_0 = inputs.isel(time=[0]) + print(input_0.time.item()) + prediction_0 = model.predict(input_0) + prediction_0.to_zarr(output_path) + for i_time in range(1, len(inputs.time)): + input_ = inputs.isel(time=[i_time]) + print(input_.time.item()) + prediction = model.predict(input_) + prediction.to_zarr(output_path, mode="a", append_dim="time") + + +if __name__ == "__main__": + if len(sys.argv[1]) > 1: + config_path = sys.argv[1] + else: + config_path = None + config = get_config(config_path) + main(config) diff --git a/projects/cloud_ml/scripts/surface_reference_zarr.py b/projects/cloud_ml/scripts/surface_reference_zarr.py new file mode 100644 index 0000000000..0f77108cc2 --- /dev/null +++ b/projects/cloud_ml/scripts/surface_reference_zarr.py @@ -0,0 +1,67 @@ +import xarray as xr +import numpy as np +import fsspec +from vcm.catalog import catalog as CATALOG +from vcm.fv3.metadata import standardize_fv3_diagnostics +from dask.diagnostics import ProgressBar + + +ENTRY = "10day_c48_PIRE_ccnorm_gfsphysics_15min_may2023" +VARIABLES = ["DSWRFsfc_coarse", "DLWRFsfc_coarse", "USWRFsfc_coarse", "PRATEsfc_coarse"] +RENAME = { + "DSWRFsfc": "override_for_time_adjusted_total_sky_downward_shortwave_flux_at_surface", # noqa: E501 + "DLWRFsfc": "override_for_time_adjusted_total_sky_downward_longwave_flux_at_surface", # noqa: E501 + "NSWRFsfc": "override_for_time_adjusted_total_sky_net_shortwave_flux_at_surface", +} +OUTPUT_PATH = "gs://vcm-ml-intermediate/2021-03-fine-res-surface-radiative-fluxes/10day-PIRE-ccnorm-coarsened-surface-radiative-fluxes-precip.zarr" # noqa: E501 + +timestep_seconds = 900.0 +m_per_mm = 1 / 1000 + + +def add_total_precipitation(ds: xr.Dataset) -> xr.Dataset: + total_precipitation = ds["PRATEsfc"] * m_per_mm * timestep_seconds + total_precipitation = total_precipitation.assign_attrs( + {"long_name": "precipitation increment to land surface", "units": "m"} + ) + ds["total_precipitation"] = total_precipitation + return ds.drop_vars("PRATEsfc") + + +def add_net_shortwave(ds: xr.Dataset) -> xr.Dataset: + net_shortwave = ds["DSWRFsfc"] - ds["USWRFsfc"] + net_shortwave = net_shortwave.assign_attrs( + { + "long_name": "net shortwave radiative flux at surface (downward)", + "units": "W/m^2", + } + ) + ds["NSWRFsfc"] = net_shortwave + return ds.drop_vars("USWRFsfc") + + +def cast_to_double(ds: xr.Dataset) -> xr.Dataset: + new_ds = {} + for name in ds.data_vars: + if ds[name].values.dtype != np.float64: + new_ds[name] = ( + ds[name] + .astype(np.float64, casting="same_kind") + .assign_attrs(ds[name].attrs) + ) + else: + new_ds[name] = ds[name] + return xr.Dataset(new_ds).assign_attrs(ds.attrs) + + +if __name__ == "__main__": + ds = ( + standardize_fv3_diagnostics(CATALOG[ENTRY].to_dask()[VARIABLES]) + .pipe(add_net_shortwave) + .pipe(add_total_precipitation) + .pipe(cast_to_double) + .rename(RENAME) + ) + with ProgressBar(): + mapper = fsspec.get_mapper(OUTPUT_PATH) + ds.to_zarr(mapper, consolidated=True) diff --git a/projects/cloud_ml/scripts/training_data_config.yaml b/projects/cloud_ml/scripts/training_data_config.yaml new file mode 100644 index 0000000000..fa0f6bf1b5 --- /dev/null +++ b/projects/cloud_ml/scripts/training_data_config.yaml @@ -0,0 +1,15 @@ +coarse_nudged_path: gs://vcm-ml-experiments/cloud-ml/2023-06-06/cloud-ml-prognostic-run-nudge-to-fine-baseline/fv3gfs_run +coarse_state_zarr: state_pre_radiation.zarr +fine_keys: + - 10day_c48_PIRE_ccnorm_gfsphysics_15min_may2023 + - 10day_c48_PIRE_ccnorm_restarts_as_zarr_may2023 +fine_rename: + liq_wat: cloud_water_mixing_ratio + ice_wat: cloud_ice_mixing_ratio + cld_amt: cloud_amount +coarse_rename: + air_temperature_pre_radiation: air_temperature + specific_humidity_pre_radiation: specific_humidity + pressure_thickness_of_atmospheric_layer_pre_radiation: pressure_thickness_of_atmospheric_layer + cloud_ice_mixing_ratio_pre_radiation: cloud_ice_mixing_ratio_coarse +output_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/cloud-ml-training-data.zarr \ No newline at end of file diff --git a/projects/cloud_ml/scripts/training_data_zarr.py b/projects/cloud_ml/scripts/training_data_zarr.py new file mode 100644 index 0000000000..68db030571 --- /dev/null +++ b/projects/cloud_ml/scripts/training_data_zarr.py @@ -0,0 +1,93 @@ +import sys +import xarray as xr +import os +from dask.diagnostics import ProgressBar +import yaml + +from vcm.catalog import catalog as CATALOG +from vcm.fv3.metadata import standardize_fv3_diagnostics +from vcm import convert_timestamps + + +GRID = CATALOG["grid/c48"].to_dask() +MASK = CATALOG["landseamask/c48"].to_dask() +COORD_VARS = ["x", "y", "z", "tile"] +OUTPUT_CHUNKS = {"time": 1, "tile": 6} + + +def get_fine_ds(keys, fine_rename): + datasets = [] + for key in keys: + dataset = CATALOG[key].to_dask() + if isinstance(dataset.time[0].item(), str): + dataset = dataset.assign_coords({"time": convert_timestamps(dataset.time)}) + dataset = standardize_fv3_diagnostics(dataset) + if "pfull" in dataset.dims: + dataset = dataset.rename({"pfull": "z"}) + datasets.append(dataset) + ds = xr.merge(datasets) + ds_out = xr.Dataset() + for restart_name, python_name in fine_rename.items(): + ds_out[python_name] = ds[restart_name] + return ds_out.drop_vars(COORD_VARS) + + +def get_coarse_ds(path, zarr, coarse_rename): + ds = xr.open_zarr(os.path.join(path, zarr)) + ds_out = xr.Dataset() + for old_name, new_name in coarse_rename.items(): + ds_out[new_name] = ds[old_name] + return ds_out + + +def subset_times(coarse, fine): + common_times = xr.DataArray( + data=sorted(list(set(coarse.time.values).intersection(set(fine.time.values)))), + dims=["time"], + ) + return coarse.sel(time=common_times), fine.sel(time=common_times) + + +def rechunk(ds, chunks=OUTPUT_CHUNKS): + for var in ds: + ds[var].encoding = {} + return ds.unify_chunks().chunk(chunks) + + +def get_config(path): + if path is not None: + with open(path, "r") as f: + return yaml.safe_load(f) + else: + return None + + +def main(config): + """ + fv3fit needs training data in a particular form; + create suitable zarr of inputs and outputs + """ + + fine = get_fine_ds(keys=config["fine_keys"], fine_rename=config["fine_rename"],) + coarse = get_coarse_ds( + path=config["coarse_nudged_path"], + zarr=config["coarse_state_zarr"], + coarse_rename=config["coarse_rename"], + ) + coarse, fine = subset_times(coarse, fine) + assert ( + len(set(coarse.data_vars).intersection(fine.data_vars)) == 0 + ), "non-overlapping names" + merged = xr.merge([fine, coarse, GRID, MASK]) + merged = rechunk(merged) + output_path = config["output_path"] + with ProgressBar(): + print(f'Number of timesteps: {merged.sizes["time"]}.') + print(f"Data variables: {[var for var in merged.data_vars]}") + merged.to_zarr(output_path, consolidated=True) + + +if __name__ == "__main__": + config_path = sys.argv[1] + config = get_config(config_path) + main(config) diff --git a/projects/cloud_ml/scripts/upload-squashed-models/squash_thresholds.yaml b/projects/cloud_ml/scripts/upload-squashed-models/squash_thresholds.yaml new file mode 100644 index 0000000000..ccb82ad587 --- /dev/null +++ b/projects/cloud_ml/scripts/upload-squashed-models/squash_thresholds.yaml @@ -0,0 +1,12 @@ +squash_thresholds: + - 0.000 + - 0.010 + - 0.030 + - 0.050 + - 0.065 + - 0.080 + - 0.100 + - 0.125 + - 0.150 + - 0.200 + - 0.250 diff --git a/projects/cloud_ml/scripts/upload-squashed-models/squashed_output_model.yaml b/projects/cloud_ml/scripts/upload-squashed-models/squashed_output_model.yaml new file mode 100644 index 0000000000..65aaab3d92 --- /dev/null +++ b/projects/cloud_ml/scripts/upload-squashed-models/squashed_output_model.yaml @@ -0,0 +1,8 @@ +base_model_path: gs://vcm-ml-experiments/cloud-ml/2023-06-07/train-cloud-ml-dense/trained_models/dense-seed-0 +squashing: + - squash_by_name: cloud_amount + additional_squash_target_names: + - cloud_water_mixing_ratio + - cloud_ice_mixing_ratio + squash_threshold: 0.0 + squash_to: 0.0 diff --git a/projects/cloud_ml/scripts/upload-squashed-models/upload.sh b/projects/cloud_ml/scripts/upload-squashed-models/upload.sh new file mode 100755 index 0000000000..1370ed77c6 --- /dev/null +++ b/projects/cloud_ml/scripts/upload-squashed-models/upload.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +N_SEEDS=$1 +OUTPUT_ROOT=$2 +SQUASH_FILE=${3:-"squash_thresholds.yaml"} + +MODEL_CONFIG="squashed_output_model.yaml" +TMP_CONFIG="/tmp/tmp_config.yaml" +readarray -t SQUASH_THRESHOLDS < <(yq '.squash_thresholds[]' ${SQUASH_FILE}) + +for ((SEED=0;SEED<$N_SEEDS;SEED++)) +do + for SQUASH_THRESHOLD in "${SQUASH_THRESHOLDS[@]}" + do + cp ${MODEL_CONFIG} ${TMP_CONFIG} + yq -yi --arg SEED "${SEED}" '.base_model_path |= (sub("seed-0"; "seed-\($SEED)"))' ${TMP_CONFIG} + yq -yi --arg SQUASH_THRESHOLD "${SQUASH_THRESHOLD}" '.squashing[].squash_threshold |= ($SQUASH_THRESHOLD | tonumber)' ${TMP_CONFIG} + gsutil cp ${TMP_CONFIG} "${OUTPUT_ROOT}/squashed-models/dense-seed-${SEED}/squash-threshold-"$SQUASH_THRESHOLD"/${MODEL_CONFIG}" + done +done + +rm $TMP_CONFIG \ No newline at end of file