Skip to content

Commit

Permalink
clear batch run notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@clkruse
clkruse committed Dec 5, 2022
1 parent e368057 commit d1739d3
Showing 1 changed file with 42 additions and 168 deletions.
210 changes: 42 additions & 168 deletions notebooks/batch_plastic_run.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -12,7 +12,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -22,7 +22,7 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -62,7 +62,7 @@
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -71,7 +71,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -86,28 +86,9 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea, North\n",
"Approx DL Time: 3.16 hours\n",
"Number of DL Tiles to process: 1582\n",
"Wrote dlkeys to ../data/boundaries/dlkeys/korea,_north_pop_0.25_dlkeys.txt\n",
"Korea, South\n",
"Approx DL Time: 2.67 hours\n",
"Number of DL Tiles to process: 1333\n",
"Wrote dlkeys to ../data/boundaries/dlkeys/korea,_south_pop_0.25_dlkeys.txt\n",
"New Caledonia\n",
"Approx DL Time: 0.13 hours\n",
"Number of DL Tiles to process: 64\n",
"Wrote dlkeys to ../data/boundaries/dlkeys/new_caledonia_pop_0.25_dlkeys.txt\n"
]
}
],
"outputs": [],
"source": [
"roi_failures = {}\n",
"for country in country_list:\n",
Expand All @@ -125,72 +106,9 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea North\n",
"Read 1582 dlkeys from ../data/boundaries/dlkeys/korea_north_pop_0.25_dlkeys.txt\n",
"Estimated cost for run: $25.60\n",
"Got patch product earthrise:korea_north_v0.0.11_2021-01-01_2022-07-01\n",
"Model spectrogram_v0.0.11_2021-07-13 found in DLStorage.\n",
"Model weak_labels_3.0_batchnorm found in DLStorage.\n",
"Creating product earthrise:korea_north_patch_weak_labels_3.0_2021-01-01_2022-07-01_stride_8_patches\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1582/1582 [03:01<00:00, 8.70it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea South\n",
"Read 1333 dlkeys from ../data/boundaries/dlkeys/korea_south_pop_0.25_dlkeys.txt\n",
"Estimated cost for run: $21.57\n",
"Got patch product earthrise:korea_south_v0.0.11_2021-01-01_2022-07-01\n",
"Model spectrogram_v0.0.11_2021-07-13 found in DLStorage.\n",
"Model weak_labels_3.0_batchnorm found in DLStorage.\n",
"Creating product earthrise:korea_south_patch_weak_labels_3.0_2021-01-01_2022-07-01_stride_8_patches\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1333/1333 [02:42<00:00, 8.21it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"New Caledonia\n",
"Existing product found.\n",
"Deleting db97e4b985ef52fa21d7e83dd688047fada1a0f2:earthrise:new_caledonia_patch_weak_labels_3.0_2021-01-01_2022-07-01_stride_8_patches\n",
"Read 64 dlkeys from ../data/boundaries/dlkeys/new_caledonia_pop_0.25_dlkeys.txt\n",
"Estimated cost for run: $1.04\n",
"Got patch product earthrise:new_caledonia_v0.0.11_2021-01-01_2022-07-01\n",
"Model spectrogram_v0.0.11_2021-07-13 found in DLStorage.\n",
"Model weak_labels_3.0_batchnorm found in DLStorage.\n",
"Creating product earthrise:new_caledonia_patch_weak_labels_3.0_2021-01-01_2022-07-01_stride_8_patches\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 64/64 [00:08<00:00, 7.23it/s]\n"
]
}
],
"outputs": [],
"source": [
"from scripts import step_2_deploy\n",
"detect_failures = {}\n",
Expand All @@ -209,55 +127,9 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea North\n",
"Creating product earthrise:korea_north_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1582/1582 [03:12<00:00, 8.20it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea South\n",
"Creating product earthrise:korea_south_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 1333/1333 [02:32<00:00, 8.74it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"New Caledonia\n",
"Creating product earthrise:new_caledonia_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 50/50 [00:06<00:00, 7.79it/s]\n"
]
}
],
"outputs": [],
"source": [
"candidate_failures = {}\n",
"for country in [c for c in country_list if c not in detect_failures.keys() and c not in roi_failures.keys()]:\n",
Expand Down Expand Up @@ -289,22 +161,9 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Korea North\n",
"korea_north_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n",
"Korea South\n",
"korea_south_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n",
"New Caledonia\n",
"new_caledonia_v0.0.11_2021-01-01_2022-07-01_blobs_thresh_0.6_min-sigma_5.0_area-thresh_0.0025_band-median\n"
]
}
],
"outputs": [],
"source": [
"query_failures = {}\n",
"for country in country_list:\n",
Expand All @@ -327,21 +186,9 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading Korea North\n",
"No candidates found for 0.0.11\n",
"Korea North failed: list index out of range\n",
"Downloading Korea South\n",
"Downloading New Caledonia\n"
]
}
],
"outputs": [],
"source": [
"download_failures = {}\n",
"for country in country_list:\n",
Expand Down Expand Up @@ -380,6 +227,33 @@
"run_title = 'caledonia_and_korea'\n",
"combined_df.to_file(f'../data/model_outputs/candidate_sites/0.0.11/{run_title}.geojson', driver='GeoJSON')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"run_title = 'remainder_batch_run'\n",
"\n",
"gdf = gpd.read_file(f'../data/model_outputs/candidate_sites/0.0.11/{run_title}.geojson')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# remove duplicates from gdf\n",
"print(len(gdf))\n",
"gdf = gdf.drop_duplicates(subset=['id'])\n",
"print(len(gdf))\n",
"gdf.to_file(f'../data/model_outputs/candidate_sites/0.0.11/{run_title}.geojson', driver='GeoJSON')"
]
}
],
"metadata": {
Expand Down

0 comments on commit d1739d3

Please sign in to comment.