Merge pull request #107 from networktocode/release-v2.0.4

Release v2.0.4

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # Default owner(s) of all files in this repository
-* @chadell @glennmatthews @pke11y @carbonarok
+* @chadell @glennmatthews @pke11y

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.0.4 - 2021-11-04
+
+### Fixed
+
+- #94 - Improve Geo service error handling.
+- #97 - Fix Readme image URLs.
+- #98 - Add handling for `Lumen` notification with Alt Circuit ID.
+- #99 - Extend `Zayo` Html parser to handle different table headers.
+- #102 - Add `Equinix` provider.
+- #104 - Use a local locations DB to map city to timezone as first option, keeping API as fallback option.
+- #105 - Extend `Colt` parser to support multiple `Maintenance` statuses.
+
 ## v2.0.3 - 2021-10-01
 
 ### Added

README.md

@@ -42,7 +42,7 @@ You can leverage this library in your automation framework to process circuit ma
 5. Each `Parser` class supports one or a set of related data types, and implements the `Parser.parse()` method used to retrieve a `Dict` with the relevant keys/values.
 
 <p align="center">
-<img src="https://raw.githubusercontent.com/nautobot/nautobot-plugin-circuit-maintenance/develop/docs/images/new_workflow.png" width="800" class="center">
+<img src="https://raw.githubusercontent.com/networktocode/circuit-maintenance-parser/develop/docs/images/new_workflow.png" width="800" class="center">
 </p>
 
 By default, there is a `GenericProvider` that support a `SimpleProcessor` using the standard `ICal` `Parser`, being the easiest path to start using the library in case the provider uses the reference iCalendar standard.
@@ -63,6 +63,7 @@ By default, there is a `GenericProvider` that support a `SimpleProcessor` using
 - AquaComms
 - Cogent
 - Colt
+- Equinix
 - GTT
 - HGC
 - Lumen
@@ -306,3 +307,7 @@ The project is following Network to Code software development guidelines and is
 
 For any questions or comments, please check the [FAQ](FAQ.md) first and feel free to swing by the [Network to Code slack channel](https://networktocode.slack.com/) (channel #networktocode).
 Sign up [here](http://slack.networktocode.com/)
+
+## License notes
+
+This library uses a Basic World Cities Database by Pareto Software, LLC, the owner of Simplemaps.com: The Provider offers a Basic World Cities Database free of charge. This database is licensed under the Creative Commons Attribution 4.0 license as described at: https://creativecommons.org/licenses/by/4.0/.

circuit_maintenance_parser/__init__.py

@@ -10,6 +10,7 @@
     AWS,
     Cogent,
     Colt,
+    Equinix,
     EUNetworks,
     GTT,
     HGC,
@@ -33,6 +34,7 @@
     AWS,
     Cogent,
     Colt,
+    Equinix,
     EUNetworks,
     GTT,
     HGC,

circuit_maintenance_parser/parser.py

@@ -16,6 +16,7 @@
 from circuit_maintenance_parser.errors import ParserError
 from circuit_maintenance_parser.output import Status, Impact, CircuitImpact
 from circuit_maintenance_parser.constants import EMAIL_HEADER_SUBJECT, EMAIL_HEADER_DATE
+from circuit_maintenance_parser.utils import Geolocator
 
 # pylint: disable=no-member
 
@@ -33,6 +34,8 @@ class Parser(BaseModel, extra=Extra.forbid):
     # _data_types are used to match the Parser to to each type of DataPart
     _data_types = ["text/plain", "plain"]
 
+    _geolocator = Geolocator()
+
     @classmethod
     def get_data_types(cls) -> List[str]:
         """Return the expected data type."""

circuit_maintenance_parser/parsers/cogent.py

@@ -7,7 +7,6 @@
 from bs4.element import ResultSet  # type: ignore
 
 from circuit_maintenance_parser.parser import Html, Impact, CircuitImpact, Status
-from circuit_maintenance_parser.utils import city_timezone
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +47,7 @@ def parse_div(self, divs: ResultSet, data: Dict):  # pylint: disable=too-many-lo
                 elif line.startswith("Cogent customers receiving service"):
                     match = re.search(r"[^Cogent].*?((\b[A-Z][a-z\s-]+)+, ([A-Za-z-]+[\s-]))", line)
                     if match:
-                        parsed_timezone = city_timezone(match.group(1).strip())
+                        parsed_timezone = self._geolocator.city_timezone(match.group(1).strip())
                         local_timezone = timezone(parsed_timezone)
                         # set start time using the local city timezone
                         start = datetime.strptime(start_str, "%I:%M %p %d/%m/%Y")

circuit_maintenance_parser/parsers/colt.py

@@ -4,66 +4,18 @@
 import re
 import csv
 import io
-import base64
 
-from icalendar import Calendar  # type: ignore
-
-from circuit_maintenance_parser.parser import Csv
-from circuit_maintenance_parser.errors import ParserError
+from dateutil import parser
 from circuit_maintenance_parser.output import Status, Impact, CircuitImpact
-from circuit_maintenance_parser.parser import ICal
+from circuit_maintenance_parser.parser import EmailSubjectParser, Csv
 
 logger = logging.getLogger(__name__)
 
 # pylint: disable=too-many-branches
 
 
-class ICalParserColt1(ICal):
-    """Colt Notifications Parser based on ICal notifications."""
-
-    def parse(self, raw: bytes):
-        """Method that returns a list of Maintenance objects."""
-        result = []
-
-        # iCalendar data sometimes comes encoded with base64
-        # TODO: add a test case
-        try:
-            gcal = Calendar.from_ical(base64.b64decode(raw))
-        except ValueError:
-            gcal = Calendar.from_ical(raw)
-
-        if not gcal:
-            raise ParserError("Not a valid iCalendar data received")
-
-        for component in gcal.walk():
-            if component.name == "VEVENT":
-                maintenance_id = ""
-                account = ""
-
-                summary_match = re.search(
-                    r"^.*?[-]\s(?P<maintenance_id>CRQ[\S]+).*?,\s*(?P<account>\d+)$", str(component.get("SUMMARY"))
-                )
-                if summary_match:
-                    maintenance_id = summary_match.group("maintenance_id")
-                    account = summary_match.group("account")
-
-                data = {
-                    "account": account,
-                    "maintenance_id": maintenance_id,
-                    "status": Status("CONFIRMED"),
-                    "start": round(component.get("DTSTART").dt.timestamp()),
-                    "end": round(component.get("DTEND").dt.timestamp()),
-                    "stamp": round(component.get("DTSTAMP").dt.timestamp()),
-                    "summary": str(component.get("SUMMARY")),
-                    "sequence": int(component.get("SEQUENCE")),
-                }
-                result.append(data)
-
-        return result
-
-
 class CsvParserColt1(Csv):
-    """Colt Notifications partial parser for circuit-ID's in CSV notifications."""
+    """Colt Notifications partial parser in CSV notifications."""
 
     @staticmethod
     def parse_csv(raw):
@@ -73,4 +25,37 @@ def parse_csv(raw):
             parsed_csv = csv.DictReader(csv_data, dialect=csv.excel_tab)
             for row in parsed_csv:
                 data["circuits"].append(CircuitImpact(impact=Impact("OUTAGE"), circuit_id=row["Circuit ID"].strip()))
+                if not data.get("account"):
+                    search = re.search(r"\d+", row["OCN"].strip())
+                    if search:
+                        data["account"] = search.group()
+        return [data]
+
+
+class SubjectParserColt1(EmailSubjectParser):
+    """Subject parser for Colt notifications."""
+
+    def parse_subject(self, subject):
+        """Parse subject.
+
+        Example:
+        - [ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 24/10/2021 04:00:00 GMT - 24/10/2021 11:00:00 GMT is about to START
+        - [ EXTERNAL ] MAINTENANCE ALERT: CRQ1-12345678 31/10/2021 00:00:00 GMT - 31/10/2021 07:30:00 GMT - COMPLETED
+        """
+        data = {}
+        search = re.search(
+            r"\[.+\](.+).+?(CRQ\w+-\w+).+?(\d+/\d+/\d+\s\d+:\d+:\d+\s[A-Z]+).+?(\d+/\d+/\d+\s\d+:\d+:\d+\s[A-Z]+).+?([A-Z]+)",
+            subject,
+        )
+        if search:
+            data["maintenance_id"] = search.group(2)
+            data["start"] = self.dt2ts(parser.parse(search.group(3)))
+            data["end"] = self.dt2ts(parser.parse(search.group(4)))
+            if search.group(5) == "START":
+                data["status"] = Status("IN-PROCESS")
+            elif search.group(5) == "COMPLETED":
+                data["status"] = Status("COMPLETED")
+            else:
+                data["status"] = Status("CONFIRMED")
+            data["summary"] = subject
         return [data]

circuit_maintenance_parser/parsers/equinix.py

@@ -0,0 +1,116 @@
+"""Circuit Maintenance Parser for Equinix Email Notifications."""
+from typing import Any, Dict, List
+import re
+
+from bs4.element import ResultSet  # type: ignore
+from dateutil import parser
+
+from circuit_maintenance_parser.output import Impact
+from circuit_maintenance_parser.parser import Html, EmailSubjectParser, Status
+
+
+class HtmlParserEquinix(Html):
+    """Custom Parser for HTML portion of Equinix circuit maintenance notifications."""
+
+    def parse_html(self, soup: ResultSet) -> List[Dict]:
+        """Parse an equinix circuit maintenance email.
+
+        Args:
+            soup (ResultSet): beautiful soup object containing the html portion of an email.
+
+        Returns:
+            Dict: The data dict containing circuit maintenance data.
+        """
+        data: Dict[str, Any] = {"circuits": list()}
+
+        impact = self._parse_b(soup.find_all("b"), data)
+        self._parse_table(soup.find_all("th"), data, impact)
+        return [data]
+
+    @staticmethod
+    def _isascii(string):
+        """Python 3.6 compatible way to determine if string is only english characters.
+
+        Args:
+            string (str): string to test if only ascii chars.
+
+        Returns:
+            bool: Returns True if string is ascii only, returns false if the string contains extended unicode characters.
+        """
+        try:
+            string.encode("ascii")
+            return True
+        except UnicodeEncodeError:
+            return False
+
+    def _parse_b(self, b_elements, data):
+        """Parse the <b> elements from the notification to capture start and end times, description, and impact.
+
+        Args:
+            b_elements (): resulting soup object with all <b> elements
+            data (Dict): data from the circuit maintenance
+
+        Returns:
+            impact (Status object): impact of the maintenance notification (used in the parse table function to assign an impact for each circuit).
+        """
+        for b_elem in b_elements:
+            if "UTC:" in b_elem:
+                raw_time = b_elem.next_sibling
+                # for non english equinix notifications
+                # english section is usually at the bottom
+                # this skips the non english line at the top
+                if not self._isascii(raw_time):
+                    continue
+                start_end_time = raw_time.split("-")
+                if len(start_end_time) == 2:
+                    data["start"] = self.dt2ts(parser.parse(raw_time.split("-")[0].strip()))
+                    data["end"] = self.dt2ts(parser.parse(raw_time.split("-")[1].strip()))
+            # all circuits in the notification share the same impact
+            if "IMPACT:" in b_elem:
+                impact_line = b_elem.next_sibling
+                if "No impact to your service" in impact_line:
+                    impact = Impact.NO_IMPACT
+                elif "There will be service interruptions" in impact_line.next_sibling.text:
+                    impact = Impact.OUTAGE
+        return impact
+
+    def _parse_table(self, theader_elements, data, impact):  # pylint: disable=no-self-use
+        for th_elem in theader_elements:
+            if "Account #" in th_elem:
+                circuit_table = th_elem.find_parent("table")
+                for tr_elem in circuit_table.find_all("tr"):
+                    if tr_elem.find(th_elem):
+                        continue
+                    circuit_info = list(tr_elem.find_all("td"))
+                    if circuit_info:
+                        account, _, circuit = circuit_info  # pylint: disable=unused-variable
+                        data["circuits"].append(
+                            {"circuit_id": circuit.text, "impact": impact,}
+                        )
+                        data["account"] = account.text
+
+
+class SubjectParserEquinix(EmailSubjectParser):
+    """Parse the subject of an equinix circuit maintenance email. The subject contains the maintenance ID and status."""
+
+    def parse_subject(self, subject: str) -> List[Dict]:
+        """Parse the Equinix Email subject for summary and status.
+
+        Args:
+            subject (str): subject of email
+            e.g. 'Scheduled software upgrade in metro connect platform-SG Metro Area Network Maintenance -19-OCT-2021 [5-212760022356]'.
+
+
+        Returns:
+            List[Dict]: Returns the data object with summary and status fields.
+        """
+        data = {}
+        maintenance_id = re.search(r"\[(.*)\]$", subject)
+        if maintenance_id:
+            data["maintenance_id"] = maintenance_id[1]
+        data["summary"] = subject.strip().replace("\n", "")
+        if "COMPLETED" in subject:
+            data["status"] = Status.COMPLETED
+        if "SCHEDULED" in subject or "REMINDER" in subject:
+            data["status"] = Status.CONFIRMED
+        return [data]

circuit_maintenance_parser/parsers/lumen.py

@@ -100,7 +100,15 @@ def parse_tables(self, tables: ResultSet, data: Dict):
                         data["status"] = "CONFIRMED"
 
                     data_circuit = {}
-                    data_circuit["circuit_id"] = cells[idx + 1].string
+
+                    # The table can include "Circuit ID" or "Alt Circuit ID" as columns +1 and +2.
+                    # Use the Circuit ID if available, else the Alt Circuit ID if available
+                    circuit_id = cells[idx + 1].string
+                    if circuit_id in ("_", "N/A"):
+                        circuit_id = cells[idx + 2].string
+                    if circuit_id not in ("_", "N/A"):
+                        data_circuit["circuit_id"] = circuit_id
+
                     impact = cells[idx + 6].string
                     if "outage" in impact.lower():
                         data_circuit["impact"] = Impact("OUTAGE")

circuit_maintenance_parser/parsers/zayo.py

@@ -61,13 +61,16 @@ def parse_tables(self, tables: ResultSet, data: Dict):
         circuits = []
         for table in tables:
             head_row = table.find_all("th")
-            if len(head_row) < 5 or [self.clean_line(line) for line in head_row[:5]] != [
-                "Circuit Id",
-                "Expected Impact",
-                "A Location CLLI",
-                "Z Location CLLI",
-                "Legacy Circuit Id",
-            ]:
+            if len(head_row) < 5:
+                logger.warning("Less table headers than expected: %s", head_row)
+                continue
+
+            table_headers = [self.clean_line(line) for line in head_row[:5]]
+            expected_headers_ref = (
+                ["Circuit Id", "Expected Impact", "A Location CLLI", "Z Location CLLI", "Legacy Circuit Id",],
+                ["Circuit Id", "Expected Impact", "A Location Address", "Z Location Address", "Legacy Circuit Id",],
+            )
+            if all(table_headers != expected_headers for expected_headers in expected_headers_ref):
                 logger.warning("Table headers are not as expected: %s", head_row)
                 continue
 

circuit_maintenance_parser/provider.py

@@ -19,7 +19,8 @@
 from circuit_maintenance_parser.parsers.aquacomms import HtmlParserAquaComms1, SubjectParserAquaComms1
 from circuit_maintenance_parser.parsers.aws import SubjectParserAWS1, TextParserAWS1
 from circuit_maintenance_parser.parsers.cogent import HtmlParserCogent1
-from circuit_maintenance_parser.parsers.colt import ICalParserColt1, CsvParserColt1
+from circuit_maintenance_parser.parsers.colt import CsvParserColt1, SubjectParserColt1
+from circuit_maintenance_parser.parsers.equinix import HtmlParserEquinix, SubjectParserEquinix
 from circuit_maintenance_parser.parsers.gtt import HtmlParserGTT1
 from circuit_maintenance_parser.parsers.hgc import HtmlParserHGC1, HtmlParserHGC2, SubjectParserHGC1
 from circuit_maintenance_parser.parsers.lumen import HtmlParserLumen1
@@ -192,11 +193,20 @@ class Colt(GenericProvider):
     """Cogent provider custom class."""
 
     _processors: List[GenericProcessor] = [
-        CombinedProcessor(data_parsers=[ICalParserColt1, CsvParserColt1]),
+        CombinedProcessor(data_parsers=[EmailDateParser, CsvParserColt1, SubjectParserColt1]),
     ]
     _default_organizer = "[email protected]"
 
 
+class Equinix(GenericProvider):
+    """Equinix provider custom class."""
+
+    _processors: List[GenericProcessor] = [
+        CombinedProcessor(data_parsers=[HtmlParserEquinix, SubjectParserEquinix, EmailDateParser]),
+    ]
+    _default_organizer = "[email protected]"
+
+
 class EUNetworks(GenericProvider):
     """EUNetworks provider custom class."""