Skip to content

Commit

Permalink
Add some minor changes to IOS parser (#67)
Browse files Browse the repository at this point in the history
* handle calibration sections within DFO IOS files.

* drop .vscode from gitignore

* move import geographiclib from if

* add geo group to test workflow

* fix test_get_sources_with_modified_hash tmp_path
  • Loading branch information
JessyBarrette authored Nov 28, 2023
1 parent e2f9f28 commit 36e4f5b
Show file tree
Hide file tree
Showing 4 changed files with 17,010 additions and 16 deletions.
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@ dist/
/pycurrents
.DS_Store

.vscode/launch.json

.vscode/settings.json
/**/local_tests/*
/tests/**/*_test.nc
pdc-amundsen-conversion.log
Expand Down
17 changes: 9 additions & 8 deletions ocean_data_parser/parsers/dfo/ios_source/IosObsFile.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def get_dtype_from_ios_name(ios_name):
return float


IOS_SHELL_HEADER_SECTIONS = (
IOS_SHELL_HEADER_SECTIONS = {
"FILE",
"LOCATION",
"COMMENTS",
Expand All @@ -107,7 +107,8 @@ def get_dtype_from_ios_name(ios_name):
"HISTORY",
"DEPLOYMENT",
"RECOVERY",
)
"CALIBRATION",
}


class IosFile(object):
Expand Down Expand Up @@ -140,6 +141,7 @@ def __init__(self, filename):
self.data = None
self.deployment = None
self.recovery = None
self.calibration = None
self.obs_time = None
self.vocabulary_attributes = None
self.history = None
Expand All @@ -159,7 +161,7 @@ def __init__(self, filename):
self.status = 1

def import_data(self):
sections_available = self.get_list_of_sections()
sections_available = set(self.get_list_of_sections())
self.start_dateobj, self.start_date = self.get_date(opt="start")
self.end_dateobj, self.end_date = (
self.get_date(opt="end") if "END TIME" in self.file else (None, None)
Expand All @@ -177,12 +179,10 @@ def import_data(self):
self.deployment = self.get_section("DEPLOYMENT")
if "RECOVERY" in sections_available:
self.recovery = self.get_section("RECOVERY")
if "CALIBRATION" in sections_available:
self.calibration = self.get_section("CALIBRATION")

unparsed_sections = [
section
for section in sections_available
if section not in IOS_SHELL_HEADER_SECTIONS
]
unparsed_sections = sections_available - IOS_SHELL_HEADER_SECTIONS
if unparsed_sections:
logger.warning(
"Unknown sections: %s",
Expand Down Expand Up @@ -932,6 +932,7 @@ def _format_attribute_value(value):
**_format_attributes("location"),
**_format_attributes("deployment", "deployment_"),
**_format_attributes("recovery", "recovery_"),
"calibration": json.dumps(self.calibration) if self.calibration else None,
"comments": str(self.comments)
if self.comments
else None, # TODO missing file_remarks
Expand Down
Loading

0 comments on commit 36e4f5b

Please sign in to comment.