Merge pull request #71 from lunkwill42/feature/linkstatetask

Implement a link state monitor task

src/zino/compat.py

@@ -0,0 +1,20 @@
+# mypy: ignore-errors
+"""Compatibility layer for older versions of Python"""
+
+try:
+    from enum import StrEnum
+except ImportError:
+    # < Python 3.11
+    from enum import Enum
+
+    class StrEnum(str, Enum):
+        __str__ = str.__str__
+
+        def _generate_next_value_(name, start, count, last_values):
+            """
+            Return the lower-cased version of the member name.
+            """
+            return name.lower()
+
+
+__all__ = ["StrEnum"]

src/zino/snmp.py

@@ -59,6 +59,7 @@ class Identifier(NamedTuple):
 PySNMPVarBind = tuple[ObjectIdentity, ObjectType]
 SNMPVarBind = tuple[Identifier, Any]
 SupportedTypes = Union[univ.Integer, univ.OctetString, ObjectIdentity, ObjectType]
+SparseWalkResponse = dict[OID, dict[str, Any]]
 
 
 class SnmpError(Exception):
@@ -348,7 +349,7 @@ async def bulkwalk(self, *oid: str, max_repetitions: int = 10) -> list[MibObject
                 results.append(mib_object)
         return results
 
-    async def sparsewalk(self, *variables: Sequence[str], max_repetitions: int = 10) -> dict[OID, dict[str, Any]]:
+    async def sparsewalk(self, *variables: Sequence[str], max_repetitions: int = 10) -> SparseWalkResponse:
         """Bulkwalks and returns a "sparse" table.
 
         A sparse walk is just a walk operation that returns selected columns of table (or, from multiple tables that

src/zino/statemodels.py

@@ -1,28 +1,32 @@
 """Basic data models for keeping/serializing/deserializing Zino state"""
 import datetime
-from enum import Enum, IntEnum
+from enum import Enum
 from ipaddress import IPv4Address, IPv6Address
 from typing import Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, Field
 
+from zino.compat import StrEnum
 from zino.time import now
 
 IPAddress = Union[IPv4Address, IPv6Address]
 AlarmType = Literal["yellow", "red"]
 PortOrIPAddress = Union[int, IPAddress, AlarmType]
 
 
-class InterfaceOperState(IntEnum):
-    """Enumerates ifOperState from RFC 2863 (IF-MIB)"""
+class InterfaceState(StrEnum):
+    """Enumerates allowable interface states.  Most of these values come from ifOperState from RFC 2863 (IF-MIB), but
+    also adds internal Zino states that don't exist in the MIB.
+    """
 
-    up = 1
-    down = 2
-    testing = 3
-    unknown = 4
-    dormant = 5
-    notPresent = 6
-    lowerLayerDown = 7
+    ADMIN_DOWN = "adminDown"
+    UP = "up"
+    DOWN = "down"
+    TESTING = "testing"
+    UNKNOWN = "unknown"
+    DORMANT = "dormant"
+    NOT_PRESENT = "notPresent"
+    LOWER_LAYER_DOWN = "lowerLayerDown"
 
 
 class Port(BaseModel):
@@ -31,7 +35,7 @@ class Port(BaseModel):
     ifindex: int
     ifdescr: Optional[str] = None
     ifalias: Optional[str] = None
-    state: Optional[InterfaceOperState] = None
+    state: Optional[InterfaceState] = None
 
 
 class DeviceState(BaseModel):
@@ -40,7 +44,7 @@ class DeviceState(BaseModel):
     name: str
     enterprise_id: Optional[int] = None
     boot_time: Optional[int] = None
-    ports: Optional[Dict[int, Port]] = None
+    ports: Dict[int, Port] = {}
     alarms: Optional[Dict[AlarmType, int]] = None
 
     # This is the remaining set of potential device attributes stored in device state by the original Zino code:
@@ -162,6 +166,8 @@ def add_history(self, message: str) -> LogEntry:
 class PortStateEvent(Event):
     type: Literal["portstate"] = "portstate"
     ifindex: Optional[int] = None
+    portstate: Optional[InterfaceState] = None
+    descr: Optional[str] = None
 
 
 class BGPEvent(Event):

src/zino/tasks/__init__.py

@@ -6,7 +6,8 @@ async def run_all_tasks(device, state):
 
 def get_registered_tasks():
     from zino.tasks.juniperalarmtask import JuniperAlarmTask
+    from zino.tasks.linkstatetask import LinkStateTask
     from zino.tasks.reachabletask import ReachableTask
     from zino.tasks.vendor import VendorTask
 
-    return [ReachableTask, VendorTask, JuniperAlarmTask]
+    return [ReachableTask, VendorTask, LinkStateTask, JuniperAlarmTask]

src/zino/tasks/linkstatetask.py

@@ -0,0 +1,151 @@
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from zino.snmp import SNMP, SparseWalkResponse
+from zino.statemodels import EventState, InterfaceState, Port, PortStateEvent
+from zino.tasks.task import Task
+
+_logger = logging.getLogger(__name__)
+
+BASE_POLL_LIST = ("ifIndex", "ifDescr", "ifAlias", "ifAdminStatus", "ifOperStatus", "ifLastChange")
+
+
+@dataclass
+class BaseInterfaceRow:
+    index: int
+    descr: str
+    alias: str
+    admin_status: str
+    oper_status: str
+    last_change: int
+
+    def is_sane(self) -> bool:
+        return bool(self.index and self.descr)
+
+
+class LinkStateTask(Task):
+    """Fetches and stores state information about router ports/links.
+
+    Things Zino 1 does at this point that this implementation ignores:
+
+    1. Zino 1 would fetch and record OLD-CISCO-INTERFACES-MIB::locIfReason, but this isn't very useful for anything
+    other than very old equipment.
+
+    2. Zino 1 collects and records interface stacking/layering information from IF-MIB::ifStackTable. It was used to
+    deem an interface as either significant or insignificant for making events about.  It was used because Uninett's
+    old convention was to set interface descriptions on only the sub-unit of Juniper ports, but this is no longer the
+    case: Descriptions are mandated for both physical ports and their sub-units.
+    """
+
+    async def run(self):
+        snmp = SNMP(self.device)
+        poll_list = [("IF-MIB", column) for column in BASE_POLL_LIST]
+        attrs = await snmp.sparsewalk(*poll_list)
+        _logger.debug("%s ifattrs: %r", self.device.name, attrs)
+
+        self._update_interfaces(attrs)
+
+    def _update_interfaces(self, new_attrs: SparseWalkResponse):
+        for index, row in new_attrs.items():
+            try:
+                self._update_single_interface(row)
+            except CollectedInterfaceDataIsNotSaneError as error:
+                _logger.error(error)
+
+    def _update_single_interface(self, row: dict[str, Any]):
+        data = BaseInterfaceRow(*(row.get(attr) for attr in BASE_POLL_LIST))
+        if not data.is_sane():
+            raise CollectedInterfaceDataIsNotSaneError(self.device.name, data)
+
+        port = self._get_or_create_port(data.index)
+        port.ifdescr = data.descr
+        self._update_ifalias(port, data)
+
+        if not self._is_interface_watched(data):
+            return
+
+        self._update_state(data, port, row)
+
+    def _update_state(self, data: BaseInterfaceRow, port: Port, row: dict[str, Any]):
+        for attr in ("ifAdminStatus", "ifOperStatus"):
+            if not row.get(attr):
+                raise MissingInterfaceTableData(self.device.name, data.index, attr)
+
+        state = f"admin{data.admin_status.capitalize()}"
+        # A special tweak so that we report ports in oper-down (but admin-up) state first time we see them
+        if not port.state and data.oper_status != "up" and state != "adminDown":
+            port.state = InterfaceState.UNKNOWN
+        if state == "adminUp":
+            state = data.oper_status
+        state = InterfaceState(state)
+        if port.state and port.state != state:
+            self._make_or_update_state_event(port, state)
+        port.state = state
+
+    def _make_or_update_state_event(self, port: Port, new_state: InterfaceState):
+        event, created = self.state.events.get_or_create_event(self.device.name, port.ifindex, PortStateEvent)
+        if created:
+            event.state = EventState.OPEN
+            event.add_history("Change state to Open")
+
+        event.portstate = new_state
+        event.ifindex = port.ifindex
+        event.polladdr = self.device.address
+        event.priority = self.device.priority
+        event.descr = port.ifdescr
+
+        # this is where we need to use sysUpTime and ifLastChange to calculate a timestamp for the change
+        log = (
+            f'{event.router}: port "{port.ifdescr}" ix {port.ifindex} ({port.ifalias}) '
+            f"changed state from {port.state} to {new_state} on TIMESTAMP"
+        )
+        _logger.info(log)
+        event.add_log(log)
+
+        # at this point we should re-schedule a new job in 2 minutes to verify the state change
+
+    def _get_or_create_port(self, ifindex: int):
+        ports = self.state.devices.get(self.device.name).ports
+        if ifindex not in ports:
+            ports[ifindex] = Port(ifindex=ifindex)
+        return ports[ifindex]
+
+    def _is_interface_watched(self, data: BaseInterfaceRow):
+        # If watch pattern exists, only watch matching interfaces
+        if self.device.watchpat and not re.match(self.device.watchpat, data.descr):
+            _logger.debug("%s intf %s not watched", self.device.name, data.descr)
+            return False
+
+        # If ignore pattern exists, ignore matching interfaces
+        if self.device.ignorepat and re.match(self.device.ignorepat, data.descr):
+            _logger.debug("%s intf %s ignored", self.device.name, data.descr)
+            return False
+
+        return True
+
+    def _update_ifalias(self, port: Port, data: BaseInterfaceRow):
+        new = port.ifalias is None
+        change = data.alias != port.ifalias
+
+        if change:
+            if not new:
+                _logger.info(
+                    "%s: changing desc for %s from %r to %r", self.device.name, data.index, port.ifalias, data.alias
+                )
+            else:
+                _logger.info("%s: setting desc for %s to %s", self.device.name, data.index, data.alias)
+            port.ifalias = data.alias
+
+
+class MissingInterfaceTableData(Exception):
+    def __init__(self, router, port, variable):
+        super().__init__(f"No {variable} from {router} for port {port}")
+
+
+class CollectedInterfaceDataIsNotSaneError(Exception):
+    def __init__(self, device: str, interface: BaseInterfaceRow):
+        self.device = device
+        self.interface = interface
+        super().__init__(f"Collected interface data from {device} is not sane enough to process: {interface!r}")

tests/compat_test.py

@@ -0,0 +1,21 @@
+import sys
+
+import pytest
+
+from zino.compat import StrEnum
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 11), reason="StrEnum compat is only valid for Python < 3.11")
+def test_strenum_value_should_be_string():
+    assert str(TestEnum.NOT_PRESENT) == "notPresent"
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 11), reason="StrEnum compat is only valid for Python < 3.11")
+def test_strenum_should_be_able_to_instantiate_from_string():
+    assert TestEnum("notPresent") == TestEnum.NOT_PRESENT
+
+
+class TestEnum(StrEnum):
+    UP = "up"
+    DOWN = "down"
+    NOT_PRESENT = "notPresent"

tests/snmp_fixtures/linksdown.snmprec

@@ -0,0 +1,90 @@
+1.3.6.1.2.1.1.1.0|4x|50726f4375727665204a34393030422053776974636820323632362c207265766973696f6e20482e30382e39382c20524f4d20482e30382e303220282f73772f636f64652f6275696c642f666973682874735f30385f352929
+1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.11.2.3.7.11.45
+1.3.6.1.2.1.1.3.0|67|3307498428
+1.3.6.1.2.1.1.4.0|4x|45696e6172204c696c6c6562727967666a656c64
+1.3.6.1.2.1.1.5.0|4|buick
+1.3.6.1.2.1.1.6.0|4x|4c43266262656e
+1.3.6.1.2.1.1.7.0|2|74
+1.3.6.1.2.1.2.2.1.1.1|2|1
+1.3.6.1.2.1.2.2.1.1.2|2|2
+1.3.6.1.2.1.2.2.1.2.1|4|1
+1.3.6.1.2.1.2.2.1.2.2|4|2
+1.3.6.1.2.1.2.2.1.3.1|2|6
+1.3.6.1.2.1.2.2.1.3.2|2|6
+1.3.6.1.2.1.2.2.1.4.1|2|1514
+1.3.6.1.2.1.2.2.1.4.2|2|1514
+1.3.6.1.2.1.2.2.1.5.1|66|10000000
+1.3.6.1.2.1.2.2.1.5.2|66|10000000
+1.3.6.1.2.1.2.2.1.6.1|4x|0019bb8f4b7f
+1.3.6.1.2.1.2.2.1.6.2|4x|0019bb8f4b7e
+1.3.6.1.2.1.2.2.1.7.1|2|1
+1.3.6.1.2.1.2.2.1.7.2|2|1
+1.3.6.1.2.1.2.2.1.8.1|2|1
+1.3.6.1.2.1.2.2.1.8.2|2|2
+1.3.6.1.2.1.2.2.1.9.1|67|1737882440
+1.3.6.1.2.1.2.2.1.9.2|67|1806268995
+1.3.6.1.2.1.2.2.1.10.1|65|0
+1.3.6.1.2.1.2.2.1.10.2|65|0
+1.3.6.1.2.1.2.2.1.11.1|65|0
+1.3.6.1.2.1.2.2.1.11.2|65|0
+1.3.6.1.2.1.2.2.1.12.1|65|0
+1.3.6.1.2.1.2.2.1.12.2|65|0
+1.3.6.1.2.1.2.2.1.13.1|65|0
+1.3.6.1.2.1.2.2.1.13.2|65|0
+1.3.6.1.2.1.2.2.1.14.1|65|0
+1.3.6.1.2.1.2.2.1.14.2|65|0
+1.3.6.1.2.1.2.2.1.15.1|65|0
+1.3.6.1.2.1.2.2.1.15.2|65|0
+1.3.6.1.2.1.2.2.1.16.1|65|0
+1.3.6.1.2.1.2.2.1.16.2|65|0
+1.3.6.1.2.1.2.2.1.17.1|65|0
+1.3.6.1.2.1.2.2.1.17.2|65|0
+1.3.6.1.2.1.2.2.1.18.1|65|0
+1.3.6.1.2.1.2.2.1.18.2|65|0
+1.3.6.1.2.1.2.2.1.19.1|65|0
+1.3.6.1.2.1.2.2.1.19.2|65|0
+1.3.6.1.2.1.2.2.1.20.1|65|0
+1.3.6.1.2.1.2.2.1.20.2|65|0
+1.3.6.1.2.1.2.2.1.21.1|66|0
+1.3.6.1.2.1.2.2.1.21.2|66|0
+1.3.6.1.2.1.2.2.1.22.1|6|1.3.6.1.2.1.10.7
+1.3.6.1.2.1.2.2.1.22.2|6|1.3.6.1.2.1.10.7
+1.3.6.1.2.1.11.30.0|2|2
+1.3.6.1.2.1.31.1.1.1.1.1|4|1
+1.3.6.1.2.1.31.1.1.1.1.2|4|2
+1.3.6.1.2.1.31.1.1.1.2.1|65|0
+1.3.6.1.2.1.31.1.1.1.2.2|65|0
+1.3.6.1.2.1.31.1.1.1.3.1|65|0
+1.3.6.1.2.1.31.1.1.1.3.2|65|0
+1.3.6.1.2.1.31.1.1.1.4.1|65|0
+1.3.6.1.2.1.31.1.1.1.4.2|65|0
+1.3.6.1.2.1.31.1.1.1.5.1|65|0
+1.3.6.1.2.1.31.1.1.1.5.2|65|0
+1.3.6.1.2.1.31.1.1.1.6.1|70|0
+1.3.6.1.2.1.31.1.1.1.6.2|70|0
+1.3.6.1.2.1.31.1.1.1.7.1|70|0
+1.3.6.1.2.1.31.1.1.1.7.2|70|0
+1.3.6.1.2.1.31.1.1.1.8.1|70|0
+1.3.6.1.2.1.31.1.1.1.8.2|70|0
+1.3.6.1.2.1.31.1.1.1.9.1|70|0
+1.3.6.1.2.1.31.1.1.1.9.2|70|0
+1.3.6.1.2.1.31.1.1.1.10.1|70|0
+1.3.6.1.2.1.31.1.1.1.10.2|70|0
+1.3.6.1.2.1.31.1.1.1.11.1|70|0
+1.3.6.1.2.1.31.1.1.1.11.2|70|0
+1.3.6.1.2.1.31.1.1.1.12.1|70|0
+1.3.6.1.2.1.31.1.1.1.12.2|70|0
+1.3.6.1.2.1.31.1.1.1.13.1|70|0
+1.3.6.1.2.1.31.1.1.1.13.2|70|0
+1.3.6.1.2.1.31.1.1.1.14.1|2|1
+1.3.6.1.2.1.31.1.1.1.14.2|2|1
+1.3.6.1.2.1.31.1.1.1.15.1|66|10
+1.3.6.1.2.1.31.1.1.1.15.2|66|10
+1.3.6.1.2.1.31.1.1.1.16.1|2|1
+1.3.6.1.2.1.31.1.1.1.16.2|2|1
+1.3.6.1.2.1.31.1.1.1.17.1|2|1
+1.3.6.1.2.1.31.1.1.1.17.2|2|1
+1.3.6.1.2.1.31.1.1.1.18.1|4x|4120706f656d
+1.3.6.1.2.1.31.1.1.1.18.2|4x|66726f6d20612066616d6f7573
+1.3.6.1.2.1.31.1.1.1.19.1|67|1737882440
+1.3.6.1.2.1.31.1.1.1.19.2|67|1806268995