Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: use delphi_utils.logger instead of copied file #1488

Merged
merged 3 commits into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,5 @@ b9ceb400d9248c8271e8342275664ac5524e335d
07ed83e5768f717ab0f9a62a9209e4e2cffa058d
# style(black): format wiki acquisition
923852eafa86b8f8b182d499489249ba8f815843
# lint: trailing whitespace changes
1d5a71db152630698e8f525b007407a83e109a83
2 changes: 1 addition & 1 deletion src/acquisition/covid_hosp/common/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

# first party
import delphi.operations.secrets as secrets
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger

Columndef = namedtuple("Columndef", "csv_name sql_name dtype")

Expand Down
3 changes: 1 addition & 2 deletions src/acquisition/covidcast/csv_importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@
import pandas as pd

# first party
from delphi_utils import Nans
from delphi_utils import get_structured_logger, Nans
from delphi.utils.epiweek import delta_epiweeks
from delphi.epidata.common.covidcast_row import CovidcastRow
from delphi.epidata.common.logger import get_structured_logger

DataFrameRow = NamedTuple('DFRow', [
('geo_id', str),
Expand Down
2 changes: 1 addition & 1 deletion src/acquisition/covidcast/csv_to_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails
from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException
from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger


def get_argument_parser():
Expand Down
20 changes: 10 additions & 10 deletions src/acquisition/covidcast/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

# first party
import delphi.operations.secrets as secrets
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger
from delphi.epidata.common.covidcast_row import CovidcastRow


Expand Down Expand Up @@ -117,28 +117,28 @@ def insert_or_update_batch(self, cc_rows: List[CovidcastRow], batch_size=2**20,
get_structured_logger("insert_or_update_batch").fatal(err_msg)
raise DBLoadStateException(err_msg)

# NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and
# NOTE: `value_update_timestamp` is hardcoded to "NOW" (which is appropriate) and
# `is_latest_issue` is hardcoded to 1 (which is temporary and addressed later in this method)
insert_into_loader_sql = f'''
INSERT INTO `{self.load_table}`
(`source`, `signal`, `time_type`, `geo_type`, `time_value`, `geo_value`,
`value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`,
`value_updated_timestamp`, `value`, `stderr`, `sample_size`, `issue`, `lag`,
`is_latest_issue`, `missing_value`, `missing_stderr`, `missing_sample_size`)
VALUES
(%s, %s, %s, %s, %s, %s,
UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s,
(%s, %s, %s, %s, %s, %s,
UNIX_TIMESTAMP(NOW()), %s, %s, %s, %s, %s,
1, %s, %s, %s)
'''

# all load table entries are already marked "is_latest_issue".
# if an entry in the load table is NOT in the latest table, it is clearly now the latest value for that key (so we do nothing (thanks to INNER join)).
# if an entry *IS* in both load and latest tables, but latest table issue is newer, unmark is_latest_issue in load.
fix_is_latest_issue_sql = f'''
UPDATE
`{self.load_table}` JOIN `{self.latest_view}`
USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`)
SET `{self.load_table}`.`is_latest_issue`=0
WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue`
UPDATE
`{self.load_table}` JOIN `{self.latest_view}`
USING (`source`, `signal`, `geo_type`, `geo_value`, `time_type`, `time_value`)
SET `{self.load_table}`.`is_latest_issue`=0
WHERE `{self.load_table}`.`issue` < `{self.latest_view}`.`issue`
dshemetov marked this conversation as resolved.
Show resolved Hide resolved
'''

# TODO: consider handling cc_rows as a generator instead of a list
Expand Down
2 changes: 1 addition & 1 deletion src/acquisition/covidcast/file_archiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import shutil

# first party
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger

class FileArchiver:
"""Archives files by moving and compressing."""
Expand Down
254 changes: 0 additions & 254 deletions src/common/logger.py

This file was deleted.

2 changes: 1 addition & 1 deletion src/maintenance/covidcast_meta_cache_updater.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

# first party
from delphi.epidata.acquisition.covidcast.database import Database
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger
from delphi.epidata.client.delphi_epidata import Epidata

def get_argument_parser():
Expand Down
2 changes: 1 addition & 1 deletion src/maintenance/delete_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

# first party
from delphi.epidata.acquisition.covidcast.database import Database
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger


def get_argument_parser():
Expand Down
8 changes: 4 additions & 4 deletions src/maintenance/signal_dash_data_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# first party
import covidcast
import delphi.operations.secrets as secrets
from delphi.epidata.common.logger import get_structured_logger
from delphi_utils import get_structured_logger


LOOKBACK_DAYS_FOR_COVERAGE = 56
Expand Down Expand Up @@ -150,11 +150,11 @@ def write_coverage(

def get_enabled_signals(self) -> List[DashboardSignal]:
"""Retrieve all enabled signals from the database"""
select_statement = f'''SELECT `id`,
select_statement = f'''SELECT `id`,
`name`,
`source`,
`covidcast_signal`,
`latest_coverage_update`,
`latest_coverage_update`,
`latest_status_update`
FROM `{Database.SIGNAL_TABLE_NAME}`
WHERE `enabled`
Expand Down Expand Up @@ -208,7 +208,7 @@ def get_coverage(dashboard_signal: DashboardSignal) -> List[DashboardSignalCover
lambda x: pd.to_datetime(Week(x // 100, x % 100).startdate()))

signal_coverage_list = []

for _, row in count_by_geo_type_df.iterrows():
signal_coverage = DashboardSignalCoverage(
signal_id=dashboard_signal.db_id,
Expand Down
Loading
Loading