Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement a link state monitor task #71

Merged
merged 19 commits into from
Oct 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/zino/compat.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# mypy: ignore-errors
"""Compatibility layer for older versions of Python"""

try:
from enum import StrEnum
except ImportError:
# < Python 3.11
from enum import Enum

class StrEnum(str, Enum):
__str__ = str.__str__

def _generate_next_value_(name, start, count, last_values):
"""
Return the lower-cased version of the member name.
"""
return name.lower()

Check warning on line 17 in src/zino/compat.py

View check run for this annotation

Codecov / codecov/patch

src/zino/compat.py#L17

Added line #L17 was not covered by tests


__all__ = ["StrEnum"]
3 changes: 2 additions & 1 deletion src/zino/snmp.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ class Identifier(NamedTuple):
PySNMPVarBind = tuple[ObjectIdentity, ObjectType]
SNMPVarBind = tuple[Identifier, Any]
SupportedTypes = Union[univ.Integer, univ.OctetString, ObjectIdentity, ObjectType]
SparseWalkResponse = dict[OID, dict[str, Any]]


class SNMP:
Expand Down Expand Up @@ -288,7 +289,7 @@ async def bulkwalk(self, *oid: str, max_repetitions: int = 10) -> list[MibObject
results.append(mib_object)
return results

async def sparsewalk(self, *variables: Sequence[str], max_repetitions: int = 10) -> dict[OID, dict[str, Any]]:
async def sparsewalk(self, *variables: Sequence[str], max_repetitions: int = 10) -> SparseWalkResponse:
"""Bulkwalks and returns a "sparse" table.

A sparse walk is just a walk operation that returns selected columns of table (or, from multiple tables that
Expand Down
30 changes: 18 additions & 12 deletions src/zino/statemodels.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,32 @@
"""Basic data models for keeping/serializing/deserializing Zino state"""
import datetime
from enum import Enum, IntEnum
from enum import Enum
from ipaddress import IPv4Address, IPv6Address
from typing import Dict, List, Literal, Optional, Union

from pydantic import BaseModel, Field

from zino.compat import StrEnum
from zino.time import now

IPAddress = Union[IPv4Address, IPv6Address]
AlarmType = Literal["yellow", "red"]
PortOrIPAddress = Union[int, IPAddress, AlarmType]


class InterfaceOperState(IntEnum):
"""Enumerates ifOperState from RFC 2863 (IF-MIB)"""
class InterfaceState(StrEnum):
"""Enumerates allowable interface states. Most of these values come from ifOperState from RFC 2863 (IF-MIB), but
also adds internal Zino states that don't exist in the MIB.
"""

up = 1
down = 2
testing = 3
unknown = 4
dormant = 5
notPresent = 6
lowerLayerDown = 7
ADMIN_DOWN = "adminDown"
UP = "up"
DOWN = "down"
TESTING = "testing"
UNKNOWN = "unknown"
DORMANT = "dormant"
NOT_PRESENT = "notPresent"
LOWER_LAYER_DOWN = "lowerLayerDown"


class Port(BaseModel):
Expand All @@ -31,7 +35,7 @@ class Port(BaseModel):
ifindex: int
ifdescr: Optional[str] = None
ifalias: Optional[str] = None
state: Optional[InterfaceOperState] = None
state: Optional[InterfaceState] = None


class DeviceState(BaseModel):
Expand All @@ -40,7 +44,7 @@ class DeviceState(BaseModel):
name: str
enterprise_id: Optional[int] = None
boot_time: Optional[int] = None
ports: Optional[Dict[int, Port]] = None
ports: Dict[int, Port] = {}
alarms: Optional[Dict[AlarmType, int]] = None

# This is the remaining set of potential device attributes stored in device state by the original Zino code:
Expand Down Expand Up @@ -162,6 +166,8 @@ def add_history(self, message: str) -> LogEntry:
class PortStateEvent(Event):
type: Literal["portstate"] = "portstate"
ifindex: Optional[int] = None
portstate: Optional[InterfaceState] = None
descr: Optional[str] = None


class BGPEvent(Event):
Expand Down
3 changes: 2 additions & 1 deletion src/zino/tasks/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ async def run_all_tasks(device, state):

def get_registered_tasks():
from zino.tasks.juniperalarmtask import JuniperAlarmTask
from zino.tasks.linkstatetask import LinkStateTask
from zino.tasks.reachabletask import ReachableTask
from zino.tasks.vendor import VendorTask

return [ReachableTask, VendorTask, JuniperAlarmTask]
return [ReachableTask, VendorTask, LinkStateTask, JuniperAlarmTask]
151 changes: 151 additions & 0 deletions src/zino/tasks/linkstatetask.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
import logging
import re
from dataclasses import dataclass
from typing import Any

from zino.snmp import SNMP, SparseWalkResponse
from zino.statemodels import EventState, InterfaceState, Port, PortStateEvent
from zino.tasks.task import Task

_logger = logging.getLogger(__name__)

BASE_POLL_LIST = ("ifIndex", "ifDescr", "ifAlias", "ifAdminStatus", "ifOperStatus", "ifLastChange")


@dataclass
class BaseInterfaceRow:
index: int
descr: str
alias: str
admin_status: str
oper_status: str
last_change: int

def is_sane(self) -> bool:
return bool(self.index and self.descr)


class LinkStateTask(Task):
"""Fetches and stores state information about router ports/links.
Things Zino 1 does at this point that this implementation ignores:
1. Zino 1 would fetch and record OLD-CISCO-INTERFACES-MIB::locIfReason, but this isn't very useful for anything
other than very old equipment.
2. Zino 1 collects and records interface stacking/layering information from IF-MIB::ifStackTable. It was used to
deem an interface as either significant or insignificant for making events about. It was used because Uninett's
old convention was to set interface descriptions on only the sub-unit of Juniper ports, but this is no longer the
case: Descriptions are mandated for both physical ports and their sub-units.
"""

async def run(self):
snmp = SNMP(self.device)
poll_list = [("IF-MIB", column) for column in BASE_POLL_LIST]
attrs = await snmp.sparsewalk(*poll_list)
_logger.debug("%s ifattrs: %r", self.device.name, attrs)

self._update_interfaces(attrs)

def _update_interfaces(self, new_attrs: SparseWalkResponse):
for index, row in new_attrs.items():
try:
self._update_single_interface(row)
except CollectedInterfaceDataIsNotSaneError as error:
_logger.error(error)

def _update_single_interface(self, row: dict[str, Any]):
data = BaseInterfaceRow(*(row.get(attr) for attr in BASE_POLL_LIST))
if not data.is_sane():
raise CollectedInterfaceDataIsNotSaneError(self.device.name, data)

port = self._get_or_create_port(data.index)
port.ifdescr = data.descr
self._update_ifalias(port, data)

if not self._is_interface_watched(data):
return

Check warning on line 67 in src/zino/tasks/linkstatetask.py

View check run for this annotation

Codecov / codecov/patch

src/zino/tasks/linkstatetask.py#L67

Added line #L67 was not covered by tests

self._update_state(data, port, row)

def _update_state(self, data: BaseInterfaceRow, port: Port, row: dict[str, Any]):
for attr in ("ifAdminStatus", "ifOperStatus"):
if not row.get(attr):
raise MissingInterfaceTableData(self.device.name, data.index, attr)

state = f"admin{data.admin_status.capitalize()}"
# A special tweak so that we report ports in oper-down (but admin-up) state first time we see them
if not port.state and data.oper_status != "up" and state != "adminDown":
port.state = InterfaceState.UNKNOWN
if state == "adminUp":
state = data.oper_status
state = InterfaceState(state)
if port.state and port.state != state:
self._make_or_update_state_event(port, state)
port.state = state
stveit marked this conversation as resolved.
Show resolved Hide resolved

def _make_or_update_state_event(self, port: Port, new_state: InterfaceState):
event, created = self.state.events.get_or_create_event(self.device.name, port.ifindex, PortStateEvent)
if created:
event.state = EventState.OPEN
event.add_history("Change state to Open")

event.portstate = new_state
event.ifindex = port.ifindex
event.polladdr = self.device.address
event.priority = self.device.priority
event.descr = port.ifdescr

# this is where we need to use sysUpTime and ifLastChange to calculate a timestamp for the change
log = (
f'{event.router}: port "{port.ifdescr}" ix {port.ifindex} ({port.ifalias}) '
f"changed state from {port.state} to {new_state} on TIMESTAMP"
)
_logger.info(log)
event.add_log(log)

# at this point we should re-schedule a new job in 2 minutes to verify the state change

def _get_or_create_port(self, ifindex: int):
ports = self.state.devices.get(self.device.name).ports
if ifindex not in ports:
ports[ifindex] = Port(ifindex=ifindex)
return ports[ifindex]

def _is_interface_watched(self, data: BaseInterfaceRow):
# If watch pattern exists, only watch matching interfaces
if self.device.watchpat and not re.match(self.device.watchpat, data.descr):
_logger.debug("%s intf %s not watched", self.device.name, data.descr)
return False

# If ignore pattern exists, ignore matching interfaces
if self.device.ignorepat and re.match(self.device.ignorepat, data.descr):
_logger.debug("%s intf %s ignored", self.device.name, data.descr)
return False

return True

def _update_ifalias(self, port: Port, data: BaseInterfaceRow):
new = port.ifalias is None
change = data.alias != port.ifalias

if change:
if not new:
_logger.info(

Check warning on line 134 in src/zino/tasks/linkstatetask.py

View check run for this annotation

Codecov / codecov/patch

src/zino/tasks/linkstatetask.py#L134

Added line #L134 was not covered by tests
"%s: changing desc for %s from %r to %r", self.device.name, data.index, port.ifalias, data.alias
)
else:
_logger.info("%s: setting desc for %s to %s", self.device.name, data.index, data.alias)
port.ifalias = data.alias
stveit marked this conversation as resolved.
Show resolved Hide resolved


class MissingInterfaceTableData(Exception):
def __init__(self, router, port, variable):
super().__init__(f"No {variable} from {router} for port {port}")


class CollectedInterfaceDataIsNotSaneError(Exception):
def __init__(self, device: str, interface: BaseInterfaceRow):
self.device = device
self.interface = interface
super().__init__(f"Collected interface data from {device} is not sane enough to process: {interface!r}")
21 changes: 21 additions & 0 deletions tests/compat_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import sys

import pytest

from zino.compat import StrEnum


@pytest.mark.skipif(sys.version_info >= (3, 11), reason="StrEnum compat is only valid for Python < 3.11")
def test_strenum_value_should_be_string():
assert str(TestEnum.NOT_PRESENT) == "notPresent"


@pytest.mark.skipif(sys.version_info >= (3, 11), reason="StrEnum compat is only valid for Python < 3.11")
def test_strenum_should_be_able_to_instantiate_from_string():
assert TestEnum("notPresent") == TestEnum.NOT_PRESENT


class TestEnum(StrEnum):
UP = "up"
DOWN = "down"
NOT_PRESENT = "notPresent"
90 changes: 90 additions & 0 deletions tests/snmp_fixtures/linksdown.snmprec
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
1.3.6.1.2.1.1.1.0|4x|50726f4375727665204a34393030422053776974636820323632362c207265766973696f6e20482e30382e39382c20524f4d20482e30382e303220282f73772f636f64652f6275696c642f666973682874735f30385f352929
1.3.6.1.2.1.1.2.0|6|1.3.6.1.4.1.11.2.3.7.11.45
1.3.6.1.2.1.1.3.0|67|3307498428
1.3.6.1.2.1.1.4.0|4x|45696e6172204c696c6c6562727967666a656c64
1.3.6.1.2.1.1.5.0|4|buick
1.3.6.1.2.1.1.6.0|4x|4c43266262656e
1.3.6.1.2.1.1.7.0|2|74
1.3.6.1.2.1.2.2.1.1.1|2|1
1.3.6.1.2.1.2.2.1.1.2|2|2
1.3.6.1.2.1.2.2.1.2.1|4|1
1.3.6.1.2.1.2.2.1.2.2|4|2
1.3.6.1.2.1.2.2.1.3.1|2|6
1.3.6.1.2.1.2.2.1.3.2|2|6
1.3.6.1.2.1.2.2.1.4.1|2|1514
1.3.6.1.2.1.2.2.1.4.2|2|1514
1.3.6.1.2.1.2.2.1.5.1|66|10000000
1.3.6.1.2.1.2.2.1.5.2|66|10000000
1.3.6.1.2.1.2.2.1.6.1|4x|0019bb8f4b7f
1.3.6.1.2.1.2.2.1.6.2|4x|0019bb8f4b7e
1.3.6.1.2.1.2.2.1.7.1|2|1
1.3.6.1.2.1.2.2.1.7.2|2|1
1.3.6.1.2.1.2.2.1.8.1|2|1
1.3.6.1.2.1.2.2.1.8.2|2|2
1.3.6.1.2.1.2.2.1.9.1|67|1737882440
1.3.6.1.2.1.2.2.1.9.2|67|1806268995
1.3.6.1.2.1.2.2.1.10.1|65|0
1.3.6.1.2.1.2.2.1.10.2|65|0
1.3.6.1.2.1.2.2.1.11.1|65|0
1.3.6.1.2.1.2.2.1.11.2|65|0
1.3.6.1.2.1.2.2.1.12.1|65|0
1.3.6.1.2.1.2.2.1.12.2|65|0
1.3.6.1.2.1.2.2.1.13.1|65|0
1.3.6.1.2.1.2.2.1.13.2|65|0
1.3.6.1.2.1.2.2.1.14.1|65|0
1.3.6.1.2.1.2.2.1.14.2|65|0
1.3.6.1.2.1.2.2.1.15.1|65|0
1.3.6.1.2.1.2.2.1.15.2|65|0
1.3.6.1.2.1.2.2.1.16.1|65|0
1.3.6.1.2.1.2.2.1.16.2|65|0
1.3.6.1.2.1.2.2.1.17.1|65|0
1.3.6.1.2.1.2.2.1.17.2|65|0
1.3.6.1.2.1.2.2.1.18.1|65|0
1.3.6.1.2.1.2.2.1.18.2|65|0
1.3.6.1.2.1.2.2.1.19.1|65|0
1.3.6.1.2.1.2.2.1.19.2|65|0
1.3.6.1.2.1.2.2.1.20.1|65|0
1.3.6.1.2.1.2.2.1.20.2|65|0
1.3.6.1.2.1.2.2.1.21.1|66|0
1.3.6.1.2.1.2.2.1.21.2|66|0
1.3.6.1.2.1.2.2.1.22.1|6|1.3.6.1.2.1.10.7
1.3.6.1.2.1.2.2.1.22.2|6|1.3.6.1.2.1.10.7
1.3.6.1.2.1.11.30.0|2|2
1.3.6.1.2.1.31.1.1.1.1.1|4|1
1.3.6.1.2.1.31.1.1.1.1.2|4|2
1.3.6.1.2.1.31.1.1.1.2.1|65|0
1.3.6.1.2.1.31.1.1.1.2.2|65|0
1.3.6.1.2.1.31.1.1.1.3.1|65|0
1.3.6.1.2.1.31.1.1.1.3.2|65|0
1.3.6.1.2.1.31.1.1.1.4.1|65|0
1.3.6.1.2.1.31.1.1.1.4.2|65|0
1.3.6.1.2.1.31.1.1.1.5.1|65|0
1.3.6.1.2.1.31.1.1.1.5.2|65|0
1.3.6.1.2.1.31.1.1.1.6.1|70|0
1.3.6.1.2.1.31.1.1.1.6.2|70|0
1.3.6.1.2.1.31.1.1.1.7.1|70|0
1.3.6.1.2.1.31.1.1.1.7.2|70|0
1.3.6.1.2.1.31.1.1.1.8.1|70|0
1.3.6.1.2.1.31.1.1.1.8.2|70|0
1.3.6.1.2.1.31.1.1.1.9.1|70|0
1.3.6.1.2.1.31.1.1.1.9.2|70|0
1.3.6.1.2.1.31.1.1.1.10.1|70|0
1.3.6.1.2.1.31.1.1.1.10.2|70|0
1.3.6.1.2.1.31.1.1.1.11.1|70|0
1.3.6.1.2.1.31.1.1.1.11.2|70|0
1.3.6.1.2.1.31.1.1.1.12.1|70|0
1.3.6.1.2.1.31.1.1.1.12.2|70|0
1.3.6.1.2.1.31.1.1.1.13.1|70|0
1.3.6.1.2.1.31.1.1.1.13.2|70|0
1.3.6.1.2.1.31.1.1.1.14.1|2|1
1.3.6.1.2.1.31.1.1.1.14.2|2|1
1.3.6.1.2.1.31.1.1.1.15.1|66|10
1.3.6.1.2.1.31.1.1.1.15.2|66|10
1.3.6.1.2.1.31.1.1.1.16.1|2|1
1.3.6.1.2.1.31.1.1.1.16.2|2|1
1.3.6.1.2.1.31.1.1.1.17.1|2|1
1.3.6.1.2.1.31.1.1.1.17.2|2|1
1.3.6.1.2.1.31.1.1.1.18.1|4x|4120706f656d
1.3.6.1.2.1.31.1.1.1.18.2|4x|66726f6d20612066616d6f7573
1.3.6.1.2.1.31.1.1.1.19.1|67|1737882440
1.3.6.1.2.1.31.1.1.1.19.2|67|1806268995
Loading
Loading