Multiple Watchers on a single host

README.md

@@ -24,6 +24,10 @@ Watcher stores topology events/state to show historical network state, whereas T
 | Extended IPv4 Reachability (new) | 135 |
 | IPv6 Reachability                | 236 |  
 
+### Network architecture  
+Number of watchers is equal to the number of IS-IS areas and each Watcher is placed in individual network namespace. IS-IS LSDB sits in watcher's namespace and doesn't interact with other Watchers keeping it isolated.  
+![](./docs/GRE_FRR_individual_instances.png)  
+
 ## Demo
 The demo shows how IS-IS watcher detected:
 * p2p links:
@@ -98,7 +102,16 @@ xpack.security.enabled: false
 ```  
 > **Note about having Elastic config commented**
 > When the Elastic output plugin fails to connect to the ELK host, it will block all other outputs and it ignores "EXPORT_TO_ELASTICSEARCH_BOOL" value from env file. Regardless of EXPORT_TO_ELASTICSEARCH_BOOL being False, it will connect to Elastic host. The solution - uncomment this portion of config in case of having running ELK.
-4. Setup GRE tunnel from the host to a network device  
+4. Generate configuration files  
+`v1.1` Includes a client for generating configurations for each Watcher for each IS-IS area. To generate individual settings - run the client with `--action add_watcher`   
+```
+sudo docker run -it --rm --user $UID -v ./:/home/watcher/watcher/ -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro vadims06/isis-watcher:v1.1 python3 ./client.py --action add_watcher
+```   
+The script will create:
+1. a folder under `watcher` folder with FRR configuration under `router` folder
+2. a containerlab configuration file with network settings
+3. an individual watcher log file in `watcher` folder.  
+
 It's needed to have a minimum one GRE tunnel to an area, which is needed to be monitored. If the IS-IS domain has multiple areas, setup one GRE in each area. It's a restriction of Link State architecture to know about new/old adjacency or link cost changes via LSPs per area basis only. To stop IS-IS routes from being installed in the host's routing table, we the following policy is applied on the watcher:
 ```bash
 # frr/config/isisd.conf
@@ -107,14 +120,20 @@ exit
 !
 ip protocol isis route-map TO_KERNEL
 ```
-
+GRE tunnel configured in Watcher namespace.  
 ```bash
 sudo modprobe ip_gre
 sudo ip tunnel add tun0 mode gre remote <router-ip> local <host-ip> dev eth0 ttl 255
 sudo ip address add <GRE tunnel ip address> dev tun0
 sudo ip link set tun0 up
 ```
-5. Setup GRE tunnel from the network device to the host. An example for Cisco
+5. Start FRR + Watcher  
+[Install](https://containerlab.srlinux.dev/install/) containerlab
+The first watcher, which uses GRE 1025 is started via the following command:   
+```
+sudo clab deploy --topo watcher/watcher1-tun1025/watcher1-tun1025.yml
+```
+6. Setup GRE tunnel from the network device to the host. An example for Cisco
 
 ```bash
 interface gigabitether0/1

client.py

@@ -0,0 +1,313 @@
+import argparse
+import ipaddress
+import shutil
+from ruamel.yaml import YAML
+from jinja2 import Environment, FileSystemLoader
+from io import StringIO
+import os
+import enum
+
+ruamel_yaml_default_mode = YAML()
+ruamel_yaml_default_mode.width = 2048  # type: ignore
+
+class ACTIONS(enum.Enum):
+    ADD_WATCHER = "add_watcher"
+    STOP_WATCHER = "stop_watcher"
+    GET_STATUS = "get_status"
+
+
+class WATCHER_CONFIG:
+    P2P_VETH_SUPERNET_W_MASK = "169.255.0.0/16"
+    WATCHER_ROOT_FOLDER = "watcher"
+    WATCHER_TEMPLATE_FOLDER_NAME = "watcher-template"
+    WATCHER_TEMPLATE_CONFIG_FILE = "config.yml"
+    ROUTER_NODE_NAME = "router"
+    ROUTER_ISIS_SYSTEMID = "49.{area_num}.{watcher_num}.{gre_num}.1111.00"
+    WATCHER_NODE_NAME = "isis-watcher"
+    def __init__(self, watcher_num):
+        self.watcher_num = watcher_num
+        # default
+        self.gre_tunnel_network_device_ip = ""
+        self.gre_tunnel_ip_w_mask_network_device = ""
+        self.gre_tunnel_ip_w_mask_watcher = ""
+        self.gre_tunnel_number = 0
+        self.isis_area_num = 1
+        self.host_interface_name = "eth0"
+
+    def gen_next_free_number(self):
+        """ Each Watcher installation has own sequense number starting from 1 """
+        return len( self.get_existed_watchers() ) + 1
+
+    @staticmethod
+    def get_existed_watchers():
+        """ Return a list of watcher folders """
+        watcher_root_folder_path = os.path.join(os.getcwd(), WATCHER_CONFIG.WATCHER_ROOT_FOLDER)
+        return [file for file in os.listdir(watcher_root_folder_path) if os.path.isdir(file) and file.startswith("watcher") and not file.endswith("template")]
+
+    @property
+    def p2p_veth_network_obj(self):
+        p2p_super_network_obj = ipaddress.ip_network(self.P2P_VETH_SUPERNET_W_MASK)
+        return self.get_nth_elem_from_iter(p2p_super_network_obj.subnets(new_prefix=24), self.watcher_num + 1)
+
+    @property
+    def p2p_veth_watcher_ip_obj(self):
+        return self.get_nth_elem_from_iter(self.p2p_veth_network_obj.hosts(), 2)
+
+    @property
+    def p2p_veth_watcher_ip_w_mask(self):
+        return f"{str(self.p2p_veth_watcher_ip_obj)}/{self.p2p_veth_network_obj.prefixlen}"
+
+    @property
+    def p2p_veth_watcher_ip_w_slash_32_mask(self):
+        return f"{str(self.p2p_veth_watcher_ip_obj)}/32"
+
+    @property
+    def p2p_veth_host_ip_obj(self):
+        return self.get_nth_elem_from_iter(self.p2p_veth_network_obj.hosts(), 1)
+
+    @property
+    def p2p_veth_host_ip_w_mask(self):
+        return f"{str(self.p2p_veth_host_ip_obj)}/{self.p2p_veth_network_obj.prefixlen}"
+
+    @property
+    def host_veth(self):
+        return f"vhost{self.gre_tunnel_number}"
+
+    @property
+    def watcher_root_folder_path(self):
+        return os.path.join(os.getcwd(), self.WATCHER_ROOT_FOLDER)
+
+    @property
+    def watcher_folder_name(self):
+        return f"watcher{self.watcher_num}-gre{self.gre_tunnel_number}"
+
+    @property
+    def watcher_folder_path(self):
+        return os.path.join(self.watcher_root_folder_path, self.watcher_folder_name)
+
+    @property
+    def watcher_template_path(self):
+        return os.path.join(self.watcher_root_folder_path, self.WATCHER_TEMPLATE_FOLDER_NAME)
+
+    @property
+    def router_template_path(self):
+        return os.path.join(self.watcher_template_path, self.ROUTER_NODE_NAME)
+
+    @property
+    def router_folder_path(self):
+        return os.path.join(self.watcher_folder_path, self.ROUTER_NODE_NAME)
+
+    @property
+    def watcher_config_template_yml(self):
+        watcher_template_path = os.path.join(self.watcher_root_folder_path, self.WATCHER_TEMPLATE_FOLDER_NAME)
+        with open(os.path.join(watcher_template_path, self.WATCHER_TEMPLATE_CONFIG_FILE)) as f:
+            return ruamel_yaml_default_mode.load(f)
+
+    @property
+    def isis_watcher_template_path(self):
+        return os.path.join(self.watcher_template_path, self.WATCHER_NODE_NAME)
+
+    @property
+    def isis_watcher_folder_path(self):
+        return os.path.join(self.watcher_folder_path, self.WATCHER_NODE_NAME)
+
+    @property
+    def netns_name(self):
+        watcher_config_yml = self.watcher_config_template_yml
+        if not watcher_config_yml.get("prefix"):
+            return f"clab-{self.watcher_folder_name}-{self.ROUTER_NODE_NAME}"
+        elif watcher_config_yml["prefix"] == "__lab-name":
+            return f"{self.watcher_folder_name}-{self.ROUTER_NODE_NAME}"
+        elif watcher_config_yml["prefix"] != "":
+            return f"{watcher_config_yml['prefix']}-{self.watcher_folder_name}-{self.ROUTER_NODE_NAME}"
+        return self.ROUTER_NODE_NAME
+
+    @staticmethod
+    def do_check_ip(ip_address_w_mask):
+        try:
+            return str(ipaddress.ip_interface(ip_address_w_mask).ip)
+        except:
+            return ""
+
+    @staticmethod
+    def _get_digit_net_mask(ip_address_w_mask):
+        return ipaddress.ip_interface(ip_address_w_mask).network.prefixlen
+
+    @staticmethod
+    def get_nth_elem_from_iter(iterator, number):
+        while number > 0:
+            value = iterator.__next__()
+            number -= 1
+        return value
+
+    @staticmethod
+    def is_network_the_same(ip_address_w_mask_1, ip_address_w_mask_2):
+        return ipaddress.ip_interface(ip_address_w_mask_1).network == ipaddress.ip_interface(ip_address_w_mask_2).network
+
+    def create_folder_with_settings(self):
+        # watcher folder
+        os.mkdir(self.watcher_folder_path)
+        # isis-watcher folder
+        watcher_logs_folder_path = os.path.join(self.watcher_root_folder_path, "logs")
+        #os.mkdir(isis_watcher_folder_path)
+        shutil.copyfile(
+            src=os.path.join(self.isis_watcher_template_path, "watcher.log"),
+            dst=os.path.join(watcher_logs_folder_path, f"{self.watcher_folder_name}.log"),
+        )
+        os.chmod(os.path.join(watcher_logs_folder_path, f"{self.watcher_folder_name}.log"), 0o755)
+        # router folder inside watcher
+        os.mkdir(self.router_folder_path)
+        for file_name in ["daemons", "isisd.log"]:
+            shutil.copyfile(
+                src=os.path.join(self.router_template_path, file_name),
+                dst=os.path.join(self.router_folder_path, file_name)
+            )
+        # Config generation
+        env = Environment(
+            loader=FileSystemLoader(self.router_template_path)
+        )
+        # frr.conf
+        frr_template = env.get_template("frr.template")
+        frr_config = frr_template.render(
+            system_id=self.ROUTER_ISIS_SYSTEMID.format(
+                area_num=str(self.isis_area_num).zfill(4),
+                watcher_num=str(self.watcher_num).zfill(4),
+                gre_num=str(self.gre_tunnel_number).zfill(4),
+        ))
+        with open(os.path.join(self.router_folder_path, "frr.conf"), "w") as f:
+            f.write(frr_config)
+        # vtysh.conf
+        vtysh_template = env.get_template("vtysh.template")
+        vtysh_config = vtysh_template.render(watcher_name=self.watcher_folder_name)
+        with open(os.path.join(self.router_folder_path, "vtysh.conf"), "w") as f:
+            f.write(vtysh_config)
+        # containerlab config
+        watcher_config_yml = self.watcher_config_template_yml
+        watcher_config_yml["name"] = self.watcher_folder_name
+        watcher_config_yml['topology']['nodes'][self.ROUTER_NODE_NAME]['ports'] = [f"{65000+self.watcher_num}:2608"]
+        watcher_config_yml['topology']['nodes'][self.WATCHER_NODE_NAME]['env']['FRR_PORT'] = str(65000+self.watcher_num)
+        watcher_config_yml['topology']['nodes']['h1']['exec'] = self.exec_cmds()
+        watcher_config_yml['topology']['links'] = [{'endpoints': [f'{self.ROUTER_NODE_NAME}:veth1', f'host:{self.host_veth}']}]
+        with open(os.path.join(self.watcher_folder_path, "config.yml"), "w") as f:
+            s = StringIO()
+            ruamel_yaml_default_mode.dump(watcher_config_yml, s)
+            f.write(s.getvalue())
+
+    def do_add_watcher_prechecks(self):
+        if os.path.exists(self.watcher_folder_path):
+            raise ValueError(f"Watcher{self.watcher_num} with GRE{self.gre_tunnel_number} already exists")
+        # TODO, check if GRE with the same tunnel destination already exist
+
+    @staticmethod
+    def do_print_banner():
+        print("""
++---------------------------+                                        
+|                           |                                        
+|  +------------+           |                                        
+|  | netns FRR  |           |                      +----------------+
+|  |            |           |                      | Network device |
+|  |  gre1   +TunnelIP------+-------------TunnelIP-+                |
+|  |            |Host int eth0                     |                |
+|  |  eth1------+-vhost1    |              Device IP                |
+|  |            |           |                      |                |
+|  +------------+           |                      |                |
+|                           |                      +----------------+
++---------------------------+                                        
+        """)
+
+    def add_watcher_dialog(self):
+        while not self.gre_tunnel_network_device_ip:
+            self.gre_tunnel_network_device_ip = self.do_check_ip(input("GRE Tunnel device IP [x.x.x.x]: "))
+        while not self.gre_tunnel_ip_w_mask_network_device:
+            self.gre_tunnel_ip_w_mask_network_device = input("GRE Tunnel IP on device with mask [x.x.x.x/yy]: ")
+            if not self.do_check_ip(self.gre_tunnel_ip_w_mask_network_device):
+                print("IP address is not correct")
+                self.gre_tunnel_ip_w_mask_network_device = ""
+            elif self._get_digit_net_mask(self.gre_tunnel_ip_w_mask_network_device) == 32:
+                print("Please provide non /32 subnet for tunnel network")
+                self.gre_tunnel_ip_w_mask_network_device = ""
+            elif self.gre_tunnel_ip_w_mask_network_device == self.gre_tunnel_network_device_ip:
+                print("Tunnel IP address shouldn't be the same as physical device IP address")
+                self.gre_tunnel_ip_w_mask_network_device = ""
+        while not self.gre_tunnel_ip_w_mask_watcher:
+            self.gre_tunnel_ip_w_mask_watcher = input("GRE Tunnel IP on Watcher with mask [x.x.x.x/yy]: ")
+            if not self.do_check_ip(self.gre_tunnel_ip_w_mask_watcher):
+                print("IP address is not correct")
+                self.gre_tunnel_ip_w_mask_watcher = ""
+            elif self._get_digit_net_mask(self.gre_tunnel_ip_w_mask_watcher) == 32:
+                print("Please provide non /32 subnet for tunnel network")
+                self.gre_tunnel_ip_w_mask_watcher = ""
+            elif not self.is_network_the_same(self.gre_tunnel_ip_w_mask_network_device, self.gre_tunnel_ip_w_mask_watcher):
+                print("Tunnel's network doesn't match")
+                self.gre_tunnel_ip_w_mask_watcher = ""
+            elif self.gre_tunnel_ip_w_mask_network_device == self.gre_tunnel_ip_w_mask_watcher:
+                print("Tunnel' IP addresses must be different on endpoints")
+                self.gre_tunnel_ip_w_mask_watcher = ""
+        while not self.gre_tunnel_number:
+            self.gre_tunnel_number = input("GRE Tunnel number: ")
+            if not self.gre_tunnel_number.isdigit():
+                print("Please provide any positive number")
+                self.gre_tunnel_number = ""
+        # ISIS settings
+        self.isis_area_num = input("IS-IS area number: ")
+        # Host interface name for NAT
+        self.host_interface_name = input("Host interface name [for GRE NAT]: ")
+
+    def exec_cmds(self):
+        return [
+            f'ip netns exec {self.netns_name} ip address add {self.p2p_veth_watcher_ip_w_mask} dev veth1',
+            f'ip netns exec {self.netns_name} ip route add {self.gre_tunnel_network_device_ip} via {str(self.p2p_veth_host_ip_obj)}',
+            f'ip address add {self.p2p_veth_host_ip_w_mask} dev {self.host_veth}',
+            f'ip netns exec {self.netns_name} ip tunnel add gre1 mode gre local {str(self.p2p_veth_watcher_ip_obj)} remote {self.gre_tunnel_network_device_ip}',
+            f'ip netns exec {self.netns_name} ip address add {self.gre_tunnel_ip_w_mask_watcher} dev gre1',
+            f'ip netns exec {self.netns_name} ip link set up dev gre1',
+            f'sudo iptables -t nat -A POSTROUTING -p gre -s {self.p2p_veth_watcher_ip_w_slash_32_mask} -d {self.gre_tunnel_network_device_ip} -o {self.host_interface_name} -j MASQUERADE',
+            f'sudo iptables -t filter -A FORWARD -p gre -s {self.p2p_veth_watcher_ip_w_slash_32_mask} -d {self.gre_tunnel_network_device_ip} -i {self.host_veth} -o {self.host_interface_name} -j ACCEPT',
+            f'sudo iptables -t filter -A FORWARD -p gre -s {self.gre_tunnel_network_device_ip} -o {self.host_veth} -i {self.host_interface_name} -j ACCEPT'
+        ]
+
+    @classmethod
+    def parse_command_args(cls, args):
+        allowed_actions = [actions.value for actions in ACTIONS]
+        if args.action not in allowed_actions:
+            raise ValueError(f"Not allowed action. Supported actions: {', '.join(allowed_actions)}")
+        watcher_num = args.watcher_num if args.watcher_num else len( cls.get_existed_watchers() ) + 1
+        watcher_obj = cls(watcher_num)
+        watcher_obj.run_command(args.action)
+
+    def run_command(self, action):
+        method = getattr(self, action)
+        return method()
+
+    def add_watcher(self):
+        self.do_print_banner()
+        self.add_watcher_dialog()
+        self.do_add_watcher_prechecks()
+        # create folder
+        self.create_folder_with_settings()
+
+    def stop_watcher(self):
+        raise NotImplementedError("Not implemented yet. Please run manually `sudo clab destroy --topo <path to config.yml>`")
+
+    def get_status(self):
+        # TODO add IS-IS neighborship status
+        raise NotImplementedError("Not implemented yet. Please run manually `sudo docker ps -f label=clab-node-name=router`")
+
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Provisioning Watcher instances for tracking IS-IS topology changes"
+    )
+    parser.add_argument(
+        "--action", required=True, help="Options: add_watcher, stop_watcher, get_status"
+    )
+    parser.add_argument(
+        "--watcher_num", required=False, default=0, help="Number of watcher"
+    )
+
+    args = parser.parse_args()
+    allowed_actions = [actions.value for actions in ACTIONS]
+    if args.action not in allowed_actions:
+        raise ValueError(f"Not allowed action. Supported actions: {', '.join(allowed_actions)}")
+    watcher_conf = WATCHER_CONFIG.parse_command_args(args)