From d302083ba26fbfc60e08f1ccf6d5f205c78ec805 Mon Sep 17 00:00:00 2001 From: unpapillon <80388812+unpapillon@users.noreply.github.com> Date: Thu, 16 Feb 2023 11:27:56 +0100 Subject: [PATCH 1/9] Update Telstra.py for new notification format (#190) * Update Telstra.py for new notification format * rename variables * Update telstra.py * add new telstra test files * Update test_e2e for telstra * Update test_parser for Telstra * fix linting for telstra parser * update regex condition * fix useless import * fix mypy lint * keep new arelion instead of Telia * fix indent * fix pylint * fix docstyle * add good result.json * fix json results * fix test_e2e * fix Telstra tests * fix compatibility with old format * fix parser organisation * fix maintenance details * fix black linting * add summary to tests * unit test fixing * improve telsrtra parser and complete tests * all test validated --------- Co-authored-by: c.radet --- circuit_maintenance_parser/parsers/telstra.py | 91 +- circuit_maintenance_parser/provider.py | 3 +- tests/unit/data/telstra/telstra8.html | 858 ++++++++++++++++++ tests/unit/data/telstra/telstra8_result.json | 16 + tests/unit/test_e2e.py | 23 + tests/unit/test_parsers.py | 7 +- 6 files changed, 995 insertions(+), 3 deletions(-) create mode 100644 tests/unit/data/telstra/telstra8.html create mode 100644 tests/unit/data/telstra/telstra8_result.json diff --git a/circuit_maintenance_parser/parsers/telstra.py b/circuit_maintenance_parser/parsers/telstra.py index be343773..86b591f3 100644 --- a/circuit_maintenance_parser/parsers/telstra.py +++ b/circuit_maintenance_parser/parsers/telstra.py @@ -1,12 +1,13 @@ """Telstra parser.""" import logging from typing import Dict, List - +import re from dateutil import parser from bs4.element import ResultSet # type: ignore from circuit_maintenance_parser.parser import Html, Impact, CircuitImpact, Status + # pylint: disable=too-many-branches @@ -73,3 +74,91 @@ def parse_tables(self, tables: ResultSet, data: Dict): # pylint: disable=too-ma # First sentence containts 'Maintenance Details:' so we skip it data["summary"] = ". ".join(sentences[1:]) break + + +class HtmlParserTelstra2(Html): + """Notifications Parser for Telstra notifications.""" + + def parse_html(self, soup): + """Execute parsing.""" + data = {} + self.parse_tables(soup.find_all("table"), data) + return [data] + + def add_maintenance_data(self, table: ResultSet, data: Dict): + """Populate data dict.""" + for strong_element in table.find_all("strong"): + if not strong_element.string: + continue + strong_text = strong_element.string.strip() + strong_sibling = strong_element.next_sibling.next_sibling + if strong_text == "Reference number": + data["maintenance_id"] = strong_sibling.string.strip() + elif strong_text == "Start time": + text_start = strong_sibling.string + regex = re.search(r"\d{2}\s[a-zA-Z]{3}\s\d{4}\s\d{2}[:]\d{2}[:]\d{2}", text_start) + if regex is not None: + start = parser.parse(regex.group()) + data["start"] = self.dt2ts(start) + else: + data["start"] = "Not defined" + elif strong_text == "End time": + text_end = strong_sibling.string + regex = re.search(r"\d{2}\s[a-zA-Z]{3}\s\d{4}\s\d{2}[:]\d{2}[:]\d{2}", text_end) + if regex is not None: + end = parser.parse(regex.group()) + data["end"] = self.dt2ts(end) + else: + data["end"] = "is not defined" + elif strong_text == "Service/s under maintenance": + data["circuits"] = [] + # TODO: This split is just an assumption of the multiple service, to be checked with more samples + impacted_circuits = strong_sibling.text.split(", ") + for circuit_id in impacted_circuits: + data["circuits"].append(CircuitImpact(impact=Impact("OUTAGE"), circuit_id=circuit_id.strip())) + elif strong_text == "Maintenance details": + sentences: List[str] = [] + for element in strong_element.next_elements: + if element.string == "Reference number": + break + if element.string and element.string not in ["\n", "", "\xa0"] + sentences: + sentences.append(element.string) + if sentences: + # First sentence containts 'Maintenance Details' so we skip it + data["summary"] = ". ".join(sentences[1:]) + + def parse_tables(self, tables: ResultSet, data: Dict): # pylint: disable=too-many-locals + """Parse Table tag.""" + for table in tables: + for p_element in table.find_all("p"): + # TODO: We should find a more consistent way to parse the status of a maintenance note + p_text = p_element.text.lower() + if "attention" in p_text: + regex = re.search("[^attention ].*", p_text.strip()) + if regex is not None: + data["account"] = regex.group() + else: + data["account"] = "not Found" + for span_element in table.find_all("span"): + span_text = span_element.text.lower() + if "planned maintenance to our network infrastructure" in span_text: + data["status"] = Status("CONFIRMED") + elif "emergency maintenance to our network infrastructure" in span_text: + data["status"] = Status("CONFIRMED") + elif "has been rescheduled" in span_text: + data["status"] = Status("RE-SCHEDULED") + elif "has been completed successfully" in span_text: + data["status"] = Status("COMPLETED") + elif ( + "did not proceed" in span_text + or "has been withdrawn" in span_text + or "has been cancelled" in span_text + ): + data["status"] = Status("CANCELLED") + elif "was unsuccessful" in span_text: + data["status"] = Status("CANCELLED") + else: + continue + break + self.add_maintenance_data(table, data) + break diff --git a/circuit_maintenance_parser/provider.py b/circuit_maintenance_parser/provider.py index 5c19505d..738431ad 100644 --- a/circuit_maintenance_parser/provider.py +++ b/circuit_maintenance_parser/provider.py @@ -35,7 +35,7 @@ SubjectParserSeaborn2, ) from circuit_maintenance_parser.parsers.sparkle import HtmlParserSparkle1 -from circuit_maintenance_parser.parsers.telstra import HtmlParserTelstra1 +from circuit_maintenance_parser.parsers.telstra import HtmlParserTelstra1, HtmlParserTelstra2 from circuit_maintenance_parser.parsers.turkcell import HtmlParserTurkcell1 from circuit_maintenance_parser.parsers.verizon import HtmlParserVerizon1 from circuit_maintenance_parser.parsers.zayo import HtmlParserZayo1, SubjectParserZayo1 @@ -330,6 +330,7 @@ class Telstra(GenericProvider): _processors: List[GenericProcessor] = [ SimpleProcessor(data_parsers=[ICal]), + CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserTelstra2]), CombinedProcessor(data_parsers=[EmailDateParser, HtmlParserTelstra1]), ] _default_organizer = "gpen@team.telstra.com" diff --git a/tests/unit/data/telstra/telstra8.html b/tests/unit/data/telstra/telstra8.html new file mode 100644 index 00000000..47a7e185 --- /dev/null +++ b/tests/unit/data/telstra/telstra8.html @@ -0,0 +1,858 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + +
+ + + + ------=_Part_312018_1139955574.1665138533169-- + \ No newline at end of file diff --git a/tests/unit/data/telstra/telstra8_result.json b/tests/unit/data/telstra/telstra8_result.json new file mode 100644 index 00000000..19e22e0e --- /dev/null +++ b/tests/unit/data/telstra/telstra8_result.json @@ -0,0 +1,16 @@ +[ + { + "account": "customer name", + "summary": "\n DCT/APT Emergency Maintenance Notification\n . \n Please be informed of emergency maintenance\n ", + "circuits": [ + { + "circuit_id": "HKG TOK EPL 9000156416", + "impact": "OUTAGE" + } + ], + "maintenance_id": "PN498484", + "status": "CONFIRMED", + "start": 1665500400, + "end": 1665525600 + } +] \ No newline at end of file diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 3a964151..9522420e 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -590,6 +590,28 @@ Path(dir_path, "data", "date", "email_date_1_result.json"), ], ), + ( + Telstra, + [ + ("html", Path(dir_path, "data", "telstra", "telstra7.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + ], + [ + Path(dir_path, "data", "telstra", "telstra7_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Telstra, + [ + ("html", Path(dir_path, "data", "telstra", "telstra8.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + ], + [ + Path(dir_path, "data", "telstra", "telstra8_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), ( Telstra, [ @@ -861,6 +883,7 @@ def test_provider_get_maintenances( Failed creating Maintenance notification for Telstra. Details: - Processor SimpleProcessor from Telstra failed due to: 1 validation error for Maintenance\naccount\n field required (type=value_error.missing) +- Processor CombinedProcessor from Telstra failed due to: None of the supported parsers for processor CombinedProcessor (EmailDateParser, HtmlParserTelstra2) was matching any of the provided data types (ical). - Processor CombinedProcessor from Telstra failed due to: None of the supported parsers for processor CombinedProcessor (EmailDateParser, HtmlParserTelstra1) was matching any of the provided data types (ical). """, ), diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index a34d4878..1117a773 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -25,7 +25,7 @@ SubjectParserSeaborn2, ) from circuit_maintenance_parser.parsers.sparkle import HtmlParserSparkle1 -from circuit_maintenance_parser.parsers.telstra import HtmlParserTelstra1 +from circuit_maintenance_parser.parsers.telstra import HtmlParserTelstra1, HtmlParserTelstra2 from circuit_maintenance_parser.parsers.turkcell import HtmlParserTurkcell1 from circuit_maintenance_parser.parsers.verizon import HtmlParserVerizon1 from circuit_maintenance_parser.parsers.zayo import SubjectParserZayo1, HtmlParserZayo1 @@ -403,6 +403,11 @@ Path(dir_path, "data", "telstra", "telstra7.html"), Path(dir_path, "data", "telstra", "telstra7_result.json"), ), + ( + HtmlParserTelstra2, + Path(dir_path, "data", "telstra", "telstra8.html"), + Path(dir_path, "data", "telstra", "telstra8_result.json"), + ), # Turkcell ( HtmlParserTurkcell1, From 5a96986a4fd06f5f508dfbb768c52dca187c7b1f Mon Sep 17 00:00:00 2001 From: Thang Do <5171823+csessh@users.noreply.github.com> Date: Thu, 9 Mar 2023 02:40:29 +1030 Subject: [PATCH 2/9] Updated documentation: Contributing section (#212) * Updated contributing section: How to debug the library locally * Updated contributing section: How to debug the library locally * Updated contributing section --- README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/README.md b/README.md index 7efa5fc1..8a429291 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,41 @@ The project is following Network to Code software development guidelines and is 4. Update the `unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created. 5. **Expose the new `Provider` class** updating the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`. +### How to debug circuit-maintenance-parser library locally + +1. `poetry install` updates the library and its dependencies locally. +2. `circuit-maintenance-parser` is now built with your recent local changes. + +If you were to add loggers or debuggers to one of the classes: + +```python +class HtmlParserZayo1(Html): + def parse_bs(self, btags: ResultSet, data: dict): + """Parse B tag.""" + raise Exception('Debugging exception') +``` + +After running `poetry install`: + +``` +-> % circuit-maintenance-parser --data-file ~/Downloads/zayo.eml --data-type email --provider-type zayo +Provider processing failed: Failed creating Maintenance notification for Zayo. +Details: +- Processor CombinedProcessor from Zayo failed due to: Debugging exception +``` + +> Note: `invoke build` will result in an error due to no Dockerfile. This is expected as the library runs simple pytest testing without a container. + +``` +-> % invoke build +Building image circuit-maintenance-parser:2.2.2-py3.8 +#1 [internal] load build definition from Dockerfile +#1 transferring dockerfile: 2B done +#1 DONE 0.0s +WARNING: failed to get git remote url: fatal: No remote configured to list refs from. +ERROR: failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to read dockerfile: open /var/lib/docker/tmp/buildkit-mount1243547759/Dockerfile: no such file or directory +``` + ## Questions For any questions or comments, please check the [FAQ](FAQ.md) first and feel free to swing by the [Network to Code slack channel](https://networktocode.slack.com/) (channel #networktocode). From 7fc8181759bfd0b2c4a9d7005a16746e25b35593 Mon Sep 17 00:00:00 2001 From: Thang Do <5171823+csessh@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:47:57 +1030 Subject: [PATCH 3/9] Issue #210: Ability to parse multiple maintenance windows from Zayo maintenance notification (#214) * Add multiple windows to Zayo maintenances * Updated unit test cases for Zayo Parser * Fixed CI/black job: replace single quotes with double quotes * Fixed CI/black job: double spaces in between classes * Fixed pydocstyle job: No blank lines allowed after function docstring --------- Co-authored-by: mkekez --- circuit_maintenance_parser/parsers/zayo.py | 65 +++++++++---------- .../data/zayo/zayo5_html_parser_result.json | 40 +++++++++++- tests/unit/data/zayo/zayo5_result.json | 40 +++++++++++- .../data/zayo/zayo6_html_parser_result.json | 44 +++++++++++-- tests/unit/data/zayo/zayo6_result.json | 44 ++++++++++++- .../data/zayo/zayo7_html_parser_result.json | 30 ++++++++- tests/unit/data/zayo/zayo7_result.json | 34 +++++++++- .../data/zayo/zayo8_html_parser_result.json | 32 ++++++++- tests/unit/data/zayo/zayo8_result.json | 46 +++++++++++-- .../data/zayo/zayo9_html_parser_result.json | 34 +++++++++- tests/unit/data/zayo/zayo9_result.json | 42 +++++++++++- tests/unit/test_parsers.py | 2 +- 12 files changed, 392 insertions(+), 61 deletions(-) diff --git a/circuit_maintenance_parser/parsers/zayo.py b/circuit_maintenance_parser/parsers/zayo.py index 4d92fc74..a6fb9c3d 100644 --- a/circuit_maintenance_parser/parsers/zayo.py +++ b/circuit_maintenance_parser/parsers/zayo.py @@ -1,6 +1,7 @@ """Zayo parser.""" import logging import re +from copy import deepcopy from typing import Dict import bs4 # type: ignore @@ -44,21 +45,30 @@ class HtmlParserZayo1(Html): def parse_html(self, soup): """Execute parsing.""" + maintenances = [] data = {} self.parse_bs(soup.find_all("b"), data) self.parse_tables(soup.find_all("table"), data) - if data: - if "status" not in data: - text = soup.get_text() - if "will be commencing momentarily" in text: - data["status"] = Status("IN-PROCESS") - elif "has been completed" in text or "has closed" in text: - data["status"] = Status("COMPLETED") - elif "has rescheduled" in text: - data["status"] = Status("RE-SCHEDULED") + if not data: + return [{}] - return [data] + if "status" not in data: + text = soup.get_text() + if "will be commencing momentarily" in text: + data["status"] = Status("IN-PROCESS") + elif "has been completed" in text or "has closed" in text: + data["status"] = Status("COMPLETED") + elif "has rescheduled" in text: + data["status"] = Status("RE-SCHEDULED") + + for maintenance_window in data.get("windows", []): + maintenance = deepcopy(data) + maintenance["start"], maintenance["end"] = maintenance_window + del maintenance["windows"] + maintenances.append(maintenance) + + return maintenances def parse_bs(self, btags: ResultSet, data: dict): """Parse B tag.""" @@ -71,41 +81,23 @@ def parse_bs(self, btags: ResultSet, data: dict): data["status"] = Status("CONFIRMED") elif "has cancelled" in line.text.lower(): data["status"] = Status("CANCELLED") - # Some Zayo notifications may include multiple activity dates. - # For lack of a better way to handle this, we consolidate these into a single extended activity range. - # - # For example, given: - # - # 1st Activity Date - # 01-Nov-2021 00:01 to 01-Nov-2021 05:00 ( Mountain ) - # 01-Nov-2021 06:01 to 01-Nov-2021 11:00 ( GMT ) - # - # 2nd Activity Date - # 02-Nov-2021 00:01 to 02-Nov-2021 05:00 ( Mountain ) - # 02-Nov-2021 06:01 to 02-Nov-2021 11:00 ( GMT ) - # - # 3rd Activity Date - # 03-Nov-2021 00:01 to 03-Nov-2021 05:00 ( Mountain ) - # 03-Nov-2021 06:01 to 03-Nov-2021 11:00 ( GMT ) - # - # our end result would be (start: "01-Nov-2021 06:01", end: "03-Nov-2021 11:00") elif "activity date" in line.text.lower(): logger.info("Found 'activity date': %s", line.text) + + if "windows" not in data: + data["windows"] = [] + for sibling in line.next_siblings: text = sibling.text if isinstance(sibling, bs4.element.Tag) else sibling logger.debug("Checking for GMT date/timestamp in sibling: %s", text) + if "( GMT )" in text: window = self.clean_line(sibling).strip("( GMT )").split(" to ") start = parser.parse(window.pop(0)) - start_ts = self.dt2ts(start) - # Keep the earliest of any listed start times - if "start" not in data or data["start"] > start_ts: - data["start"] = start_ts end = parser.parse(window.pop(0)) + start_ts = self.dt2ts(start) end_ts = self.dt2ts(end) - # Keep the latest of any listed end times - if "end" not in data or data["end"] < end_ts: - data["end"] = end_ts + data["windows"].append((start_ts, end_ts)) break elif line.text.lower().strip().startswith("reason for maintenance:"): data["summary"] = self.clean_line(line.next_sibling) @@ -148,6 +140,7 @@ def parse_tables(self, tables: ResultSet, data: Dict): "Customer Circuit ID", ], ) + if all(table_headers != expected_headers for expected_headers in expected_headers_ref): logger.warning("Table headers are not as expected: %s", head_row) continue @@ -155,6 +148,7 @@ def parse_tables(self, tables: ResultSet, data: Dict): data_rows = table.find_all("td") if len(data_rows) % 5 != 0: raise AssertionError("Table format is not correct") + number_of_circuits = int(len(data_rows) / 5) for idx in range(number_of_circuits): data_circuit = {} @@ -165,5 +159,6 @@ def parse_tables(self, tables: ResultSet, data: Dict): elif "no expected impact" in impact.lower(): data_circuit["impact"] = Impact("NO-IMPACT") circuits.append(CircuitImpact(**data_circuit)) + if circuits: data["circuits"] = circuits diff --git a/tests/unit/data/zayo/zayo5_html_parser_result.json b/tests/unit/data/zayo/zayo5_html_parser_result.json index 6aaf9397..cc13df9e 100644 --- a/tests/unit/data/zayo/zayo5_html_parser_result.json +++ b/tests/unit/data/zayo/zayo5_html_parser_result.json @@ -1,5 +1,7 @@ [ { + "maintenance_id": "TTN-0003456789", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", "circuits": [ { "circuit_id": "/OGYX/123456/ /ZYO /", @@ -10,10 +12,42 @@ "impact": "NO-IMPACT" } ], - "end": 1635937200, - "maintenance_id": "TTN-0003456789", + "status": "IN-PROCESS", "start": 1635746460, + "end": 1635764400 + }, + { + "maintenance_id": "TTN-0003456789", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", + "circuits": [ + { + "circuit_id": "/OGYX/123456/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id": "/OGYX/234567/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "status": "IN-PROCESS", + "start": 1635832860, + "end": 1635850800 + }, + { + "maintenance_id": "TTN-0003456789", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", + "circuits": [ + { + "circuit_id": "/OGYX/123456/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id": "/OGYX/234567/ /ZYO /", + "impact": "NO-IMPACT" + } + ], "status": "IN-PROCESS", - "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + "start": 1635919260, + "end": 1635937200 } ] diff --git a/tests/unit/data/zayo/zayo5_result.json b/tests/unit/data/zayo/zayo5_result.json index 34feb2ef..1da93a71 100644 --- a/tests/unit/data/zayo/zayo5_result.json +++ b/tests/unit/data/zayo/zayo5_result.json @@ -11,11 +11,49 @@ "impact": "NO-IMPACT" } ], - "end": 1635937200, + "end": 1635764400, "maintenance_id": "TTN-0003456789", "stamp": 1635918838, + "status": "IN-PROCESS", "start": 1635746460, + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + }, + { + "account": "Some Customer Inc", + "circuits": [ + { + "circuit_id": "/OGYX/123456/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id": "/OGYX/234567/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "end": 1635850800, + "maintenance_id": "TTN-0003456789", + "stamp": 1635918838, + "status": "IN-PROCESS", + "start": 1635832860, + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + }, + { + "account": "Some Customer Inc", + "circuits": [ + { + "circuit_id": "/OGYX/123456/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id": "/OGYX/234567/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "end": 1635937200, + "maintenance_id": "TTN-0003456789", + "stamp": 1635918838, "status": "IN-PROCESS", + "start": 1635919260, "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." } ] diff --git a/tests/unit/data/zayo/zayo6_html_parser_result.json b/tests/unit/data/zayo/zayo6_html_parser_result.json index 6fc7f11c..98c8e8aa 100644 --- a/tests/unit/data/zayo/zayo6_html_parser_result.json +++ b/tests/unit/data/zayo/zayo6_html_parser_result.json @@ -1,19 +1,53 @@ [ { + "maintenance_id": "TTN-0004567890", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", "circuits": [ { "circuit_id": "/OGYX/123418/ /ZYO /", "impact": "NO-IMPACT" }, { - "circuit_id": "/OGYX/123408/ /ZYO /", + "circuit_id":"/OGYX/123408/ /ZYO /", "impact": "NO-IMPACT" } ], - "end": 1635937200, - "maintenance_id": "TTN-0004567890", + "status": "COMPLETED", "start": 1635746460, + "end": 1635764400 + }, + { + "maintenance_id": "TTN-0004567890", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", + "circuits": [ + { + "circuit_id": "/OGYX/123418/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id":"/OGYX/123408/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "status": "COMPLETED", + "start": 1635832860, + "end": 1635850800 + }, + { + "maintenance_id": "TTN-0004567890", + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic.", + "circuits": [ + { + "circuit_id": "/OGYX/123418/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id":"/OGYX/123408/ /ZYO /", + "impact": "NO-IMPACT" + } + ], "status": "COMPLETED", - "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + "start": 1635919260, + "end": 1635937200 } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo6_result.json b/tests/unit/data/zayo/zayo6_result.json index 372d9776..cec054f1 100644 --- a/tests/unit/data/zayo/zayo6_result.json +++ b/tests/unit/data/zayo/zayo6_result.json @@ -7,15 +7,53 @@ "impact": "NO-IMPACT" }, { - "circuit_id": "/OGYX/123408/ /ZYO /", + "circuit_id":"/OGYX/123408/ /ZYO /", "impact": "NO-IMPACT" } ], - "end": 1635937200, + "end": 1635764400, "maintenance_id": "TTN-0004567890", "stamp": 1635936668, + "status": "COMPLETED", "start": 1635746460, + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + }, + { + "account": "Some Customer Inc", + "circuits": [ + { + "circuit_id": "/OGYX/123418/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id":"/OGYX/123408/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "end": 1635850800, + "maintenance_id": "TTN-0004567890", + "stamp": 1635936668, + "status": "COMPLETED", + "start": 1635832860, + "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." + }, + { + "account": "Some Customer Inc", + "circuits": [ + { + "circuit_id": "/OGYX/123418/ /ZYO /", + "impact": "NO-IMPACT" + }, + { + "circuit_id":"/OGYX/123408/ /ZYO /", + "impact": "NO-IMPACT" + } + ], + "end": 1635937200, + "maintenance_id": "TTN-0004567890", + "stamp": 1635936668, "status": "COMPLETED", + "start": 1635919260, "summary": "Routine Fiber splice - NO Impact is Expected to your services. This notification is to advise you that we will be entering a splice case that houses live traffic." } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo7_html_parser_result.json b/tests/unit/data/zayo/zayo7_html_parser_result.json index 5d185420..5b9703b8 100644 --- a/tests/unit/data/zayo/zayo7_html_parser_result.json +++ b/tests/unit/data/zayo/zayo7_html_parser_result.json @@ -6,10 +6,36 @@ "impact": "OUTAGE" } ], - "end": 1637067600, + "end": 1636894800, "maintenance_id": "TTN-0005432100", "start": 1636876860, "status": "COMPLETED", "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" + }, + { + "circuits": [ + { + "circuit_id": "/IPYX/100722/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1636981200, + "maintenance_id": "TTN-0005432100", + "start": 1636963260, + "status": "COMPLETED", + "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" + }, + { + "circuits": [ + { + "circuit_id": "/IPYX/100722/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1637067600, + "maintenance_id": "TTN-0005432100", + "start": 1637049660, + "status": "COMPLETED", + "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo7_result.json b/tests/unit/data/zayo/zayo7_result.json index ba5a64ec..fdfc29b7 100644 --- a/tests/unit/data/zayo/zayo7_result.json +++ b/tests/unit/data/zayo/zayo7_result.json @@ -7,11 +7,41 @@ "impact": "OUTAGE" } ], - "end": 1637067600, + "end": 1636894800, "maintenance_id": "TTN-0005432100", "stamp": 1637068032, "start": 1636876860, "status": "COMPLETED", "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" + }, + { + "account": "Example Inc.", + "circuits": [ + { + "circuit_id": "/IPYX/100722/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1636981200, + "maintenance_id": "TTN-0005432100", + "stamp": 1637068032, + "start": 1636963260, + "status": "COMPLETED", + "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" + }, + { + "account": "Example Inc.", + "circuits": [ + { + "circuit_id": "/IPYX/100722/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1637067600, + "maintenance_id": "TTN-0005432100", + "stamp": 1637068032, + "start": 1637049660, + "status": "COMPLETED", + "summary": "Zayo will implement maintenance to repair damaged fiber splice case, to prevent unplanned outages" } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo8_html_parser_result.json b/tests/unit/data/zayo/zayo8_html_parser_result.json index fa92f9e3..d9b63621 100644 --- a/tests/unit/data/zayo/zayo8_html_parser_result.json +++ b/tests/unit/data/zayo/zayo8_html_parser_result.json @@ -7,11 +7,41 @@ "impact": "OUTAGE" } ], - "end": 1641639600, + "end": 1641466800, "maintenance_id": "TTN-0001234567", "stamp": 1639526400, "start": 1641445260, "status": "RE-SCHEDULED", "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641553200, + "maintenance_id": "TTN-0001234567", + "stamp": 1639526400, + "start": 1641531660, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641639600, + "maintenance_id": "TTN-0001234567", + "stamp": 1639526400, + "start": 1641618060, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" } ] diff --git a/tests/unit/data/zayo/zayo8_result.json b/tests/unit/data/zayo/zayo8_result.json index bef1ac48..e69c24e8 100644 --- a/tests/unit/data/zayo/zayo8_result.json +++ b/tests/unit/data/zayo/zayo8_result.json @@ -3,11 +3,11 @@ "account": "Customer,inc", "circuits": [ { - "circuit_id": "/OQYX/234567/ /ZYO /", - "impact": "OUTAGE" + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" } ], - "end": 1641639600, + "end": 1641466800, "maintenance_id": "TTN-0001234567", "organizer": "mr@zayo.com", "provider": "zayo", @@ -17,5 +17,43 @@ "status": "RE-SCHEDULED", "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", "uid": "0" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641553200, + "maintenance_id": "TTN-0001234567", + "organizer": "mr@zayo.com", + "provider": "zayo", + "sequence": 1, + "stamp": 1639526400, + "start": 1641531660, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", + "uid": "0" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641639600, + "maintenance_id": "TTN-0001234567", + "organizer": "mr@zayo.com", + "provider": "zayo", + "sequence": 1, + "stamp": 1639526400, + "start": 1641618060, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", + "uid": "0" } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo9_html_parser_result.json b/tests/unit/data/zayo/zayo9_html_parser_result.json index ae36bebd..00253880 100644 --- a/tests/unit/data/zayo/zayo9_html_parser_result.json +++ b/tests/unit/data/zayo/zayo9_html_parser_result.json @@ -7,11 +7,41 @@ "impact": "OUTAGE" } ], - "end": 1641639600, + "end": 1641466800, "maintenance_id": "TTN-0001234567", "stamp": 1639526400, "start": 1641445260, "status": "RE-SCHEDULED", "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641553200, + "maintenance_id": "TTN-0001234567", + "stamp": 1639526400, + "start": 1641531660, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641639600, + "maintenance_id": "TTN-0001234567", + "stamp": 1639526400, + "start": 1641618060, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted" } -] +] \ No newline at end of file diff --git a/tests/unit/data/zayo/zayo9_result.json b/tests/unit/data/zayo/zayo9_result.json index cec6ed48..ecfccd56 100644 --- a/tests/unit/data/zayo/zayo9_result.json +++ b/tests/unit/data/zayo/zayo9_result.json @@ -7,7 +7,7 @@ "impact": "OUTAGE" } ], - "end": 1641639600, + "end": 1641466800, "maintenance_id": "TTN-0001234567", "organizer": "mr@zayo.com", "provider": "zayo", @@ -17,5 +17,43 @@ "status": "RE-SCHEDULED", "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", "uid": "0" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641553200, + "maintenance_id": "TTN-0001234567", + "organizer": "mr@zayo.com", + "provider": "zayo", + "sequence": 1, + "stamp": 1639526400, + "start": 1641531660, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", + "uid": "0" + }, + { + "account": "Customer,inc", + "circuits": [ + { + "circuit_id": "/OQYX/234567/ /ZYO /", + "impact": "OUTAGE" + } + ], + "end": 1641639600, + "maintenance_id": "TTN-0001234567", + "organizer": "mr@zayo.com", + "provider": "zayo", + "sequence": 1, + "stamp": 1639526400, + "start": 1641618060, + "status": "RE-SCHEDULED", + "summary": "Zayo Third-Party Provider will implement maintenance to perform temporary fiber relocation in order to proactively avoid outages.\"Please Note\" This is a reschedule of TTN-0002345678\r\nAs this maintenance is not under the control of Zayo it may not be possible to reschedule it, due to resources having been coordinated with the railway system (Lat: 50.12345- Lon: -119.12345)The maintenance consists of 2 nights; however, customers will only receive notification for the night their services will be impacted", + "uid": "0" } -] +] \ No newline at end of file diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index 1117a773..1be4978b 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -547,7 +547,7 @@ def test_parsers(parser_class, raw_file, results_file): assert parsed_notifications == expected_result -@pytest.mark.parametrize("parser_class", [ICal, EmailDateParser, HtmlParserZayo1, SubjectParserSeaborn1]) +@pytest.mark.parametrize("parser_class", [ICal, EmailDateParser, HtmlParserZayo1, SubjectParserZayo1]) def test_parser_no_data(parser_class): """Test parser with no data.""" with pytest.raises(ParserError): From 9586d47afe950ad67e402df0c98595c850c22246 Mon Sep 17 00:00:00 2001 From: Mark Kekez <92973272+aliex-13@users.noreply.github.com> Date: Mon, 20 Mar 2023 23:12:13 +1030 Subject: [PATCH 4/9] Allow Lumen maintenance multiple windows to be parsed (#216) * Add multiple window checks for Lumen * NRE-605 fix comments * Add unit tests with multiple windows Lumen * Fix linter issues from PR * Fix another linter typo * Fix linter pydocstyle * readme changes for more instructions on local testing * Added some more instructions and examples for readme * Update test_e2e tests for Lumen * remove date tests in e2e file for Lumen8 --------- Co-authored-by: mkekez --- README.md | 22 +++ circuit_maintenance_parser/parsers/lumen.py | 25 ++- tests/unit/data/lumen/lumen8.html | 164 ++++++++++++++++++++ tests/unit/data/lumen/lumen8_result.json | 40 +++++ tests/unit/test_e2e.py | 48 ++++++ tests/unit/test_parsers.py | 5 + 6 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 tests/unit/data/lumen/lumen8.html create mode 100644 tests/unit/data/lumen/lumen8_result.json diff --git a/README.md b/README.md index 8a429291..96011919 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,28 @@ The project is following Network to Code software development guidelines and is - The `Provider` also supports the definition of a `_include_filter` and a `_exclude_filter` to limit the notifications that are actually processed, avoiding false positive errors for notification that are not relevant. 4. Update the `unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created. 5. **Expose the new `Provider` class** updating the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`. +6. You can run some tests here to verify that your new unit tests do not cause issues with existing tests, and in general they work as expected. You can do this by running `pytest --log-cli-level=DEBUG --capture=tee-sys`. You can narrow down the tests that you want to execute with the `-k` flag. If successful, your results should look similar to the following: + +``` +-> % pytest --log-cli-level=DEBUG --capture=tee-sys -k test_parsers +...omitted debug logs... +====================================================== 99 passed, 174 deselected, 17 warnings in 10.35s ====================================================== +``` +7. Run some final CI tests locally to ensure that there is no linting/formatting issues with your changes. You should look to get a code score of 10/10. See the example below: `invoke tests --local` + +``` +-> % invoke tests --local +LOCAL - Running command black --check --diff . +All done! ✨ 🍰 ✨ +41 files would be left unchanged. +LOCAL - Running command flake8 . +LOCAL - Running command find . -name "*.py" | xargs pylint +************* Module tasks +tasks.py:4:0: W0402: Uses of a deprecated module 'distutils.util' (deprecated-module) + +-------------------------------------------------------------------- +Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) +``` ### How to debug circuit-maintenance-parser library locally diff --git a/circuit_maintenance_parser/parsers/lumen.py b/circuit_maintenance_parser/parsers/lumen.py index eb0ca468..2b835d94 100644 --- a/circuit_maintenance_parser/parsers/lumen.py +++ b/circuit_maintenance_parser/parsers/lumen.py @@ -2,6 +2,7 @@ import logging from typing import Dict +from copy import deepcopy from dateutil import parser import bs4 # type: ignore from bs4.element import ResultSet # type: ignore @@ -19,10 +20,22 @@ class HtmlParserLumen1(Html): def parse_html(self, soup): """Execute parsing.""" + maintenances = [] data = {} self.parse_spans(soup.find_all("span"), data) self.parse_tables(soup.find_all("table"), data) - return [data] + + # Iterates over multiple windows and duplicates other maintenance info to a new dictionary while also updating start and end times for the specific window. + for window in data["windows"]: + maintenance = deepcopy(data) + maintenance["start"], maintenance["end"] = window + del maintenance["windows"] + maintenances.append(maintenance) + + # Deleting the key after we are finished checking for multiple windows and duplicating data. + del data["windows"] + + return maintenances def parse_spans(self, spans: ResultSet, data: Dict): """Parse Span tag.""" @@ -56,8 +69,11 @@ def parse_spans(self, spans: ResultSet, data: Dict): data["stamp"] = self.dt2ts(stamp) break - def parse_tables(self, tables: ResultSet, data: Dict): + def parse_tables(self, tables: ResultSet, data: Dict): # pylint: disable=too-many-locals """Parse Table tag.""" + # Initialise multiple windows list that will be used in parse_html + data["windows"] = [] + circuits = [] for table in tables: cells = table.find_all("td") @@ -68,9 +84,10 @@ def parse_tables(self, tables: ResultSet, data: Dict): for idx in range(num_columns, len(cells), num_columns): if "GMT" in cells[idx].string and "GMT" in cells[idx + 1].string: start = parser.parse(cells[idx].string.split(" GMT")[0]) - data["start"] = self.dt2ts(start) + start_ts = self.dt2ts(start) end = parser.parse(cells[idx + 1].string.split(" GMT")[0]) - data["end"] = self.dt2ts(end) + end_ts = self.dt2ts(end) + data["windows"].append((start_ts, end_ts)) break elif cells[0].string == "Customer Name": diff --git a/tests/unit/data/lumen/lumen8.html b/tests/unit/data/lumen/lumen8.html new file mode 100644 index 00000000..a4ed0f5c --- /dev/null +++ b/tests/unit/data/lumen/lumen8.html @@ -0,0 +1,164 @@ + + BODY { font-family: Arial; font-size:10pt;} + + + + + + + + + + + + + + + + + + + + + + + +

+         3D"Lumen" +

+
+ +Scheduled Maintenance #: 12345678 +

+Summary: +

+Lumen intends to carry out internal maintenance within its network. This h= +as been designated as ESSENTIAL. The nature of this work is to repair fiber= + and is required in order to avoid unplanned outages from damages related t= +o natural causes. +
+
The estimated GPS location of work is:=01 +
LAT: 2 LONG: -1 +
+
An alternate/backup maintenance window has been scheduled in the event = +work is unable to be completed on the primary night. Currently no services= + are scheduled to be impacted on the alternate night. Should the alternate= + night be deemed necessary, you will receive an updated notification advisi= +ng which services will be impacted on that night. Please see the date and = +times of the alternate night below. +
+
Lumen sincerely apologizes for any inconvenience caused by this main= +tenance. +

+Updates: +

+ +2023-03-09 16:23:23 GMT - This maintenance is scheduled.
+
+ +Customer Impact: +

+

12345678-1
= +
StartEnd
2023-03-22 04:00 GMT (G= +reenwich Mean Time)2023-03-22 10:00 GMT (Greenwich Mean Time)
2023-03-22 00:00 EDT (Eastern Daylight Time)2023-03-2= +2 06:00 EDT (Eastern Daylight Time)


Maintenance Location(s): RANDOM LOCATION= +;ANOTHER RANDOM ADDRESS SOMEWHERE

= +<= +td>Outage
Customer NameCircuit IDAlt= + Circuit IDBandwidthA LocationZ LocationImpact TypeMaximum DurationOrder Number
Customer123456789N/A100 GIG122 LATELY ST PONARM MI USA992 A PRAM= +KA ST CHICAGO IL USAOutage4 hours  
CUSTOMER = + 12345678910GLAN-1234567810GIG-E LAN= +SOMETHING SOMETHING SOMETHING RANDOM LOCATION IN THE USA 4 hours  

Alternate Night:
12345678-2
StartEnd
2023= +-03-23 04:00 GMT (Greenwich Mean Time)2023-03-23 10:00 GMT (Greenw= +ich Mean Time)
2023-03-23 00:00 EDT (Eastern Daylight Time= +)2023-03-23 06:00 EDT (Eastern Daylight Time)
+

+ + +Click here to open a case for assistance on this scheduled maintenance = +via Email. + +

+Click here for immediate information on scheduled maintenances via the = +Lumen Customer Portal. + +

+Click here to manage your notification subscriptions via the Lumen Port= +al. + +

+Click here to open a case for assistance on this scheduled maintenance = +via the Lumen Customer Portal. +


+Network Change Management Team +
+Lumen +
+1-855-244-6468 option 1 +

+The information in this communication is confidential and may not be disclo= +sed to=20 +third parties or shared further without the express permission of Lumen.= \ No newline at end of file diff --git a/tests/unit/data/lumen/lumen8_result.json b/tests/unit/data/lumen/lumen8_result.json new file mode 100644 index 00000000..17b0f886 --- /dev/null +++ b/tests/unit/data/lumen/lumen8_result.json @@ -0,0 +1,40 @@ +[ + { + "account": "Customer", + "circuits": [ + { + "circuit_id": "123456789", + "impact": "OUTAGE" + }, + { + "circuit_id": "123456789", + "impact": "OUTAGE" + } + ], + "end": 1679479200, + "maintenance_id": "12345678", + "stamp": 1678379003, + "start": 1679457600, + "status": "IN-PROCESS", + "summary": "Lumen intends to carry out internal maintenance within its network. This has been designated as ESSENTIAL. The nature of this work is to repair fiber and is required in order to avoid unplanned outages from damages related to natural causes." + }, + { + "account": "Customer", + "circuits": [ + { + "circuit_id": "123456789", + "impact": "OUTAGE" + }, + { + "circuit_id": "123456789", + "impact": "OUTAGE" + } + ], + "end": 1679565600, + "maintenance_id": "12345678", + "stamp": 1678379003, + "start": 1679544000, + "status": "IN-PROCESS", + "summary": "Lumen intends to carry out internal maintenance within its network. This has been designated as ESSENTIAL. The nature of this work is to repair fiber and is required in order to avoid unplanned outages from damages related to natural causes." + } +] \ No newline at end of file diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 9522420e..bb1e5993 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -423,6 +423,54 @@ Path(dir_path, "data", "date", "email_date_1_result.json"), ], ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen5.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen5_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen6.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen6_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen7.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen7_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen8.html")), + # (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen8_result.json"), + # Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), # Megaport ( Megaport, diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index 1be4978b..f0963aaf 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -317,6 +317,11 @@ Path(dir_path, "data", "lumen", "lumen7.html"), Path(dir_path, "data", "lumen", "lumen7_result.json"), ), + ( + HtmlParserLumen1, + Path(dir_path, "data", "lumen", "lumen8.html"), + Path(dir_path, "data", "lumen", "lumen8_result.json"), + ), # Megaport ( HtmlParserMegaport1, From 8c8fe373dfd01593af87bdc08ea4e39278605e93 Mon Sep 17 00:00:00 2001 From: Christian Adell Date: Mon, 20 Mar 2023 14:31:15 +0100 Subject: [PATCH 5/9] Fix e2e test while combining data from multiple maintenaces wia single parsed data --- tests/unit/data/date/email_date_1_result.json | 6 ++---- tests/unit/test_e2e.py | 17 ++++++++++++----- tests/unit/test_parsers.py | 5 ++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/unit/data/date/email_date_1_result.json b/tests/unit/data/date/email_date_1_result.json index 6cc5fab1..3538ae20 100644 --- a/tests/unit/data/date/email_date_1_result.json +++ b/tests/unit/data/date/email_date_1_result.json @@ -1,5 +1,3 @@ -[ - { +{ "stamp": 1612172014 - } -] +} diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index bb1e5993..3a5e5067 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -463,12 +463,11 @@ Lumen, [ ("html", Path(dir_path, "data", "lumen", "lumen8.html")), - # (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), ], [ Path(dir_path, "data", "lumen", "lumen8_result.json"), - # Path(dir_path, "data", "date", "email_date_1_result.json"), ], ), # Megaport @@ -823,7 +822,7 @@ ) def test_provider_get_maintenances( provider_class, test_data_files, result_parse_files -): # pylint: disable=too-many-locals +): # pylint: disable=too-many-locals,too-many-branches """End to End tests for various Providers.""" extended_data = provider_class.get_extended_data() default_maintenance_data = {"uid": "0", "sequence": 1, "summary": ""} @@ -849,10 +848,18 @@ def test_provider_get_maintenances( for result_parse_file in result_parse_files: with open(result_parse_file, encoding="utf-8") as res_file: partial_result_data = json.load(res_file) - if not expected_result: + + # TODO: Tests assume that maintenances (multiple) will be discovered on the first parser + if not expected_result and isinstance(partial_result_data, list): expected_result = partial_result_data + + if expected_result and isinstance(partial_result_data, dict): + for _ in range(len(expected_result)): + expected_result[0].update(partial_result_data) else: - expected_result[0].update(partial_result_data[0]) + assert len(expected_result) == len(expected_result) + for i, _ in enumerate(partial_result_data): + expected_result[i].update(partial_result_data[i]) for result in expected_result: temp_res = result.copy() diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index f0963aaf..b2a2a900 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -549,7 +549,10 @@ def test_parsers(parser_class, raw_file, results_file): with open(results_file, encoding="utf-8") as res_file: expected_result = json.load(res_file) - assert parsed_notifications == expected_result + if parser_class == EmailDateParser: + assert parsed_notifications == [expected_result] + else: + assert parsed_notifications == expected_result @pytest.mark.parametrize("parser_class", [ICal, EmailDateParser, HtmlParserZayo1, SubjectParserZayo1]) From 5ce626c71850894413d5ba114b6676435488a7a8 Mon Sep 17 00:00:00 2001 From: Christian Adell Date: Mon, 20 Mar 2023 14:39:43 +0100 Subject: [PATCH 6/9] Add Verizon Parser 4 tests in e2e --- tests/unit/test_e2e.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 3a5e5067..eea067ef 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -725,6 +725,17 @@ Path(dir_path, "data", "date", "email_date_1_result.json"), ], ), + ( + Verizon, + [ + ("html", Path(dir_path, "data", "verizon", "verizon4.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + ], + [ + Path(dir_path, "data", "verizon", "verizon4_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), ( Verizon, [ From fbc4474da2a1703040b95b903068792ae484b9c3 Mon Sep 17 00:00:00 2001 From: Christian Adell Date: Mon, 20 Mar 2023 14:44:31 +0100 Subject: [PATCH 7/9] skip pylint too-many-lines --- tests/unit/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index eea067ef..b3ed99ee 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -9,7 +9,7 @@ from circuit_maintenance_parser.errors import ProviderError from circuit_maintenance_parser.constants import EMAIL_HEADER_DATE, EMAIL_HEADER_SUBJECT -# pylint: disable=duplicate-code +# pylint: disable=duplicate-code,too-many-lines from circuit_maintenance_parser.provider import ( Equinix, GenericProvider, From fda6541af232cb8d9dc7ba3179ef5d74d7c3a8ec Mon Sep 17 00:00:00 2001 From: Christian Adell Date: Mon, 20 Mar 2023 18:52:34 +0100 Subject: [PATCH 8/9] Fix assert --- tests/unit/test_e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index b3ed99ee..ab8eb39e 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -868,7 +868,7 @@ def test_provider_get_maintenances( for _ in range(len(expected_result)): expected_result[0].update(partial_result_data) else: - assert len(expected_result) == len(expected_result) + assert len(expected_result) == len(partial_result_data) for i, _ in enumerate(partial_result_data): expected_result[i].update(partial_result_data[i]) From 228600a1ed7d8b74541b465b8fb904b342ee3686 Mon Sep 17 00:00:00 2001 From: Christian Adell Date: Tue, 21 Mar 2023 14:17:39 +0100 Subject: [PATCH 9/9] Release v2.2.3 --- CHANGELOG.md | 13 +++++++++++++ pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26313593..ac362d88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v2.2.3 - 2023-03-21 + +### Changed + +- #216 - Allow Lumen maintenance multiple windows to be parsed +- #212 - Updated documentation: Contribution section +- #210 - Ability to parse multiple maintenance windows from Zayo +- #190 - Update Telstra for new notificaiton format + +### Fixed + +- #222 - Fix e22 tests when combining data from multiple maintenances + ## v2.2.2 - 2023-01-27 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 610a653a..a0b39db9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "circuit-maintenance-parser" -version = "2.2.2" +version = "2.2.3" description = "Python library to parse Circuit Maintenance notifications and return a structured data back" authors = ["Network to Code "] license = "Apache-2.0"