Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix excessive numbers of indexing metrics being created #551

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Fix excessive numbers of indexing metrics being created #551

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b453931
Don't tag pod/host on indexing metrics
omad Feb 28, 2023
8e119be
Improve naming, error checking
omad Feb 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions apps/dc_tools/odc/apps/dc_tools/add_update_products.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
from datacube import Datacube
from odc.apps.dc_tools.utils import (
update_if_exists_flag,
statsd_gauge_reporting,
statsd_setting,
report_statsd_gauge,
statsd_server,
)

Product = namedtuple("Product", ["name", "doc"])
Expand Down Expand Up @@ -125,8 +125,8 @@ def add_update_products(
@click.command("dc-sync-products")
@click.argument("csv-path", nargs=1)
@update_if_exists_flag
@statsd_setting
def cli(csv_path: str, update_if_exists: bool, statsd_setting: str):
@statsd_server
def cli(csv_path: str, update_if_exists: bool, statsd_server: str):
# Check we can connect to the Datacube
dc = datacube.Datacube(app="add_update_products")
logging.info(
Expand All @@ -138,15 +138,15 @@ def cli(csv_path: str, update_if_exists: bool, statsd_setting: str):
added, updated, failed = add_update_products(dc, csv_path, update_if_exists)

print(f"Added: {added}, Updated: {updated} and Failed: {failed}")
if statsd_setting:
statsd_gauge_reporting(
added, ["app: add_update_products", "action:added"], statsd_setting
if statsd_server:
report_statsd_gauge(
added, ["app: add_update_products", "action:added"], statsd_server
)
statsd_gauge_reporting(
failed, ["app: add_update_products", "action:failed"], statsd_setting
report_statsd_gauge(
failed, ["app: add_update_products", "action:failed"], statsd_server
)
statsd_gauge_reporting(
failed, ["app: add_update_products", "action:updated"], statsd_setting
report_statsd_gauge(
failed, ["app: add_update_products", "action:updated"], statsd_server
)

# If nothing failed then this exists with success code 0
Expand Down
18 changes: 7 additions & 11 deletions apps/dc_tools/odc/apps/dc_tools/azure_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
allow_unsafe,
archive_less_mature,
index_update_dataset,
statsd_gauge_reporting,
statsd_setting,
report_statsd_gauge,
statsd_server,
transform_stac,
update_flag,
update_if_exists_flag,
Expand Down Expand Up @@ -79,7 +79,7 @@ def dump_list_to_odc(
@update_if_exists_flag
@allow_unsafe
@transform_stac
@statsd_setting
@statsd_server
@archive_less_mature
@publish_action
@click.option(
Expand All @@ -101,7 +101,7 @@ def cli(
update_if_exists: bool,
allow_unsafe: bool,
stac: bool,
statsd_setting: str,
statsd_server: str,
archive_less_mature: bool,
publish_action: str,
account_url: str,
Expand Down Expand Up @@ -134,13 +134,9 @@ def cli(
)

print(f"Added {added} Datasets, Failed to add {failed} Datasets")
if statsd_setting:
statsd_gauge_reporting(
added, ["app:azure_to_dc", "action:added"], statsd_setting
)
statsd_gauge_reporting(
failed, ["app:azure_to_dc", "action:failed"], statsd_setting
)
if statsd_server:
report_statsd_gauge(added, ["app:azure_to_dc", "action:added"], statsd_server)
report_statsd_gauge(failed, ["app:azure_to_dc", "action:failed"], statsd_server)


if __name__ == "__main__":
Expand Down
22 changes: 10 additions & 12 deletions apps/dc_tools/odc/apps/dc_tools/cop_dem_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
limit,
update_if_exists_flag,
publish_action,
statsd_gauge_reporting,
statsd_setting,
report_statsd_gauge,
statsd_server,
)
from ._stac import stac_transform

Expand Down Expand Up @@ -205,7 +205,7 @@ def cop_dem_to_dc(
@limit
@update_if_exists_flag
@bbox
@statsd_setting
@statsd_server
@archive_less_mature
@publish_action
@click.option(
Expand All @@ -229,7 +229,7 @@ def cli(
limit,
update_if_exists,
bbox,
statsd_setting,
statsd_server,
product,
add_product,
workers,
Expand Down Expand Up @@ -266,15 +266,13 @@ def cli(
f"Added {added} Datasets, failed {failed} Datasets, skipped {skipped} Datasets"
)

if statsd_setting:
statsd_gauge_reporting(
added, ["app:cop_dem_to_dc", "action:added"], statsd_setting
if statsd_server:
report_statsd_gauge(added, ["app:cop_dem_to_dc", "action:added"], statsd_server)
report_statsd_gauge(
failed, ["app:cop_dem_to_dc", "action:failed"], statsd_server
)
statsd_gauge_reporting(
failed, ["app:cop_dem_to_dc", "action:failed"], statsd_setting
)
statsd_gauge_reporting(
skipped, ["app:cop_dem_to_dc", "action:skipped"], statsd_setting
report_statsd_gauge(
skipped, ["app:cop_dem_to_dc", "action:skipped"], statsd_server
)

if failed > 0:
Expand Down
18 changes: 9 additions & 9 deletions apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
limit,
update_if_exists_flag,
archive_less_mature,
statsd_gauge_reporting,
statsd_setting,
report_statsd_gauge,
statsd_server,
publish_action,
)
from ._stac import stac_transform
Expand Down Expand Up @@ -230,7 +230,7 @@ def esa_wc_to_dc(
type=int,
help="Number of threads to use to process, default 20",
)
@statsd_setting
@statsd_server
@click.option(
"--version",
default="2020",
Expand All @@ -244,7 +244,7 @@ def cli(
add_product,
archive_less_mature,
workers,
statsd_setting,
statsd_server,
publish_action,
version,
):
Expand Down Expand Up @@ -274,12 +274,12 @@ def cli(

print(f"Added {added} Datasets, failed {failed} Datasets")

if statsd_setting:
statsd_gauge_reporting(
added, ["app:esa_worldcover_to_dc", "action:added"], statsd_setting
if statsd_server:
report_statsd_gauge(
added, ["app:esa_worldcover_to_dc", "action:added"], statsd_server
)
statsd_gauge_reporting(
failed, ["app:esa_worldcover_to_dc", "action:failed"], statsd_setting
report_statsd_gauge(
failed, ["app:esa_worldcover_to_dc", "action:failed"], statsd_server
)

if failed > 0:
Expand Down
16 changes: 7 additions & 9 deletions apps/dc_tools/odc/apps/dc_tools/fs_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
index_update_dataset,
update_if_exists_flag,
publish_action,
statsd_setting,
statsd_gauge_reporting,
statsd_server,
report_statsd_gauge,
transform_stac,
)

Expand All @@ -31,7 +31,7 @@
@allow_unsafe
@archive_less_mature
@transform_stac
@statsd_setting
@statsd_server
@publish_action
@click.option(
"--glob",
Expand All @@ -43,7 +43,7 @@ def cli(
update_if_exists,
allow_unsafe,
stac,
statsd_setting,
statsd_server,
glob,
archive_less_mature,
publish_action,
Expand Down Expand Up @@ -87,11 +87,9 @@ def cli(
failed += 1

logging.info("Added %s and failed %s datasets.", added, failed)
if statsd_setting:
statsd_gauge_reporting(added, ["app:fs_to_dc", "action:added"], statsd_setting)
statsd_gauge_reporting(
failed, ["app:fs_to_dc", "action:failed"], statsd_setting
)
if statsd_server:
report_statsd_gauge(added, ["app:fs_to_dc", "action:added"], statsd_server)
report_statsd_gauge(failed, ["app:fs_to_dc", "action:failed"], statsd_server)


if __name__ == "__main__":
Expand Down
20 changes: 8 additions & 12 deletions apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
request_payer,
skip_check,
skip_lineage,
statsd_gauge_reporting,
statsd_setting,
report_statsd_gauge,
statsd_server,
transform_stac,
transform_stac_absolute,
update_flag,
Expand Down Expand Up @@ -106,7 +106,7 @@ def dump_to_odc(
@allow_unsafe
@skip_check
@no_sign_request
@statsd_setting
@statsd_server
@request_payer
@archive_less_mature
@publish_action
Expand All @@ -123,7 +123,7 @@ def cli(
allow_unsafe,
skip_check,
no_sign_request,
statsd_setting,
statsd_server,
request_payer,
archive_less_mature,
publish_action,
Expand Down Expand Up @@ -184,14 +184,10 @@ def cli(
print(
f"Added {added} datasets, skipped {skipped} datasets and failed {failed} datasets."
)
if statsd_setting:
statsd_gauge_reporting(added, ["app:s3_to_dc", "action:added"], statsd_setting)
statsd_gauge_reporting(
skipped, ["app:s3_to_dc", "action:skipped"], statsd_setting
)
statsd_gauge_reporting(
failed, ["app:s3_to_dc", "action:failed"], statsd_setting
)
if statsd_server:
report_statsd_gauge(added, ["app:s3_to_dc", "action:added"], statsd_server)
report_statsd_gauge(skipped, ["app:s3_to_dc", "action:skipped"], statsd_server)
report_statsd_gauge(failed, ["app:s3_to_dc", "action:failed"], statsd_server)

if failed > 0:
sys.exit(failed)
Expand Down
36 changes: 16 additions & 20 deletions apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
limit,
no_sign_request,
skip_lineage,
statsd_setting,
statsd_gauge_reporting,
statsd_server,
report_statsd_gauge,
transform_stac,
transform_stac_absolute,
archive_less_mature,
Expand Down Expand Up @@ -322,7 +322,7 @@ def queue_to_odc(
@archive
@limit
@no_sign_request
@statsd_setting
@statsd_server
@click.option(
"--odc-metadata-link",
default=None,
Expand Down Expand Up @@ -358,7 +358,7 @@ def cli(
allow_unsafe,
archive,
limit,
statsd_setting,
statsd_server,
no_sign_request,
odc_metadata_link,
record_path,
Expand Down Expand Up @@ -408,32 +408,28 @@ def cli(
result_msg = ""
if update:
result_msg += f"Updated {success} Dataset(s), "
if statsd_setting:
statsd_gauge_reporting(
success, ["app:sqs_to_dc", "action:update"], statsd_setting
if statsd_server:
report_statsd_gauge(
success, ["app:sqs_to_dc", "action:update"], statsd_server
)
elif archive:
result_msg += f"Archived {success} Dataset(s), "
if statsd_setting:
statsd_gauge_reporting(
success, ["app:sqs_to_dc", "action:archived"], statsd_setting
if statsd_server:
report_statsd_gauge(
success, ["app:sqs_to_dc", "action:archived"], statsd_server
)
else:
result_msg += f"Added {success} Dataset(s), "
if statsd_setting:
statsd_gauge_reporting(
success, ["app:sqs_to_dc", "action:added"], statsd_setting
if statsd_server:
report_statsd_gauge(
success, ["app:sqs_to_dc", "action:added"], statsd_server
)
result_msg += f"Failed {failed} Dataset(s), Skipped {skipped} Dataset(s)"
print(result_msg)

if statsd_setting:
statsd_gauge_reporting(
failed, ["app:sqs_to_dc", "action:failed"], statsd_setting
)
statsd_gauge_reporting(
skipped, ["app:sqs_to_dc", "action:skipped"], statsd_setting
)
if statsd_server:
report_statsd_gauge(failed, ["app:sqs_to_dc", "action:failed"], statsd_server)
report_statsd_gauge(skipped, ["app:sqs_to_dc", "action:skipped"], statsd_server)

if failed > 0:
sys.exit(failed)
Expand Down
Loading