From 644a7a86baace4da9607e6fa77bf1a654fb07c12 Mon Sep 17 00:00:00 2001 From: Aryaman Date: Fri, 22 Sep 2023 19:29:37 +0530 Subject: [PATCH] [feature] Added support for ZeroTier parser #135 Closes #135 --- .github/workflows/ci.yml | 2 + README.rst | 91 +++++++++++++++++- .../integrations/device/base/models.py | 31 ++++++ .../device/management/commands/__init__.py | 1 + .../device/tests/test_integration.py | 94 +++++++++++++++++++ .../migrations/0016_alter_topology_parser.py | 1 + openwisp_network_topology/settings.py | 1 + requirements.txt | 2 +- .../migrations/0001_initial.py | 1 + 9 files changed, 221 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ffcc28e..625e5802 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,8 @@ jobs: - name: Install test dependencies run: | pip install -U -r requirements-test.txt + pip install --force-reinstall \ + git+https://github.com/openwisp/openwisp-controller/@issue-606/zerotier-member-auth-ip-assign - name: Install openwisp-network-topology run: | diff --git a/README.rst b/README.rst index 5dc4f07f..90cc71ef 100644 --- a/README.rst +++ b/README.rst @@ -100,6 +100,7 @@ Available features - CNML 1.0 - OpenVPN - Wireguard + - ZeroTier - additional formats can be added by `writing custom netdiff parsers `_ * **network topology visualizer** based on @@ -340,6 +341,92 @@ Sending data for topology with RECEIVE strategy or, alternatively, a non-admin visualizer page is also available at the URL ``/topology/topology//``. +Sending data for ZeroTier topology with RECEIVE strategy +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Follow the procedure described below to setup ZeroTier topology with RECEIVE strategy. + +**Note:** In this example, the **Shared systemwide (no organization)** +option is used for the ZeroTier topology organization. You are free to +opt for any organization, as long as both the topology and the device share +the same organization, assuming the `OpenWISP controller integration +<#integration-with-openwisp-controller-and-openwisp-monitoring>`_ feature is enabled. + +1. Create topology for ZeroTier +############################### + +1. Visit ``admin/topology/topology/add`` to add a new topology. + +2. We will set the **Label** of this topology to ``ZeroTier`` and + select the topology **Format** from the dropdown as ``ZeroTier``. + +3. Select the strategy as ``RECEIVE`` from the dropdown. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-network-topology/docs/docs/zerotier-tutorial/topology-1.png + :alt: ZeroTier topology configuration example 1 + +4. Let use default **Expiration time** ``0`` and make sure **Published** option is checked. + +5. After clicking on the **Save and continue editing** button, a topology receive URL is generated. + Make sure you copy that URL for later use in the topology script. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-network-topology/docs/docs/zerotier-tutorial/topology-2.png + :alt: ZeroTier topology configuration example 2 + +2. Create a script for sending ZeroTier topology data +##################################################### + +1. Now, create a script (e.g: ``/opt/send-zt-topology.sh``) that sends + the ZeroTier topology data using a POST request. In the example script below, + we are sending the ZeroTier self-hosted controller peers data: + +.. code-block:: shell + + #!/bin/bash + # command to fetch zerotier controller peers data in json format + COMMAND="zerotier-cli peers -j" + UUID="" + KEY="" + OPENWISP_URL="https://" + $COMMAND | + # Upload the topology data to OpenWISP + curl -X POST \ + --data-binary @- \ + --header "Content-Type: text/plain" \ + $OPENWISP_URL/api/v1/network-topology/topology/$UUID/receive/?key=$KEY + +2. Add the ``/opt/send-zt-topology.sh`` script created in the previous step + to the root crontab, here's an example which sends the topology data every **5 minutes**: + +.. code-block:: shell + + # flag script as executable + chmod +x /opt/send-zt-topology.sh + +.. code-block:: shell + + # open rootcrontab + sudo crontab -e + + ## Add the following line and save + + echo */5 * * * * /opt/send-zt-topology.sh + +**Note:** When using the **ZeroTier** topology, ensure that +you use ``sudo crontab -e`` to edit the **root crontab**. This step +is essential because the ``zerotier-cli peers -j`` command requires **root privileges** +for kernel interaction, without which the command will not function correctly. + +3. Once the steps above are completed, you should see nodes and links + being created automatically, you can see the network topology graph + from the admin page of the topology change page (you have to click on + the **View topology graph** button in the upper right part of the page) + or, alternatively, a non-admin visualizer page is also available at + the URL ``/topology/topology//``. + + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-network-topology/docs/docs/zerotier-tutorial/topology-graph.png + :alt: ZeroTier topology graph example 1 + Management Commands ------------------- @@ -457,8 +544,8 @@ Integration with OpenWISP Controller and OpenWISP Monitoring If you use `OpenWISP Controller `_ or `OpenWISP Monitoring `_ -and you use OpenVPN or Wireguard for the management VPN, you can use the integration -available in ``openwisp_network_topology.integrations.device``. +and you use OpenVPN, Wireguard or ZeroTier for the management VPN, you can use +the integration available in ``openwisp_network_topology.integrations.device``. This additional and optional module provides the following features: diff --git a/openwisp_network_topology/integrations/device/base/models.py b/openwisp_network_topology/integrations/device/base/models.py index 99ff4b14..8307171b 100644 --- a/openwisp_network_topology/integrations/device/base/models.py +++ b/openwisp_network_topology/integrations/device/base/models.py @@ -36,6 +36,9 @@ class AbstractDeviceNode(UUIDModel): 'netdiff.WireguardParser': { 'auto_create': 'auto_create_wireguard', }, + 'netdiff.ZeroTierParser': { + 'auto_create': 'auto_create_zerotier', + }, 'netdiff.NetJsonParser': { 'auto_create': 'auto_create_netjsongraph', }, @@ -127,6 +130,34 @@ def auto_create_wireguard(cls, node): return return cls.save_device_node(device, node) + @classmethod + def auto_create_zerotier(cls, node): + """ + Implementation of the integration between + controller and network-topology modules + when using ZeroTier (using the `zerotier_member_id`) + """ + zerotier_member_id = node.properties.get('address') + if not zerotier_member_id: + return + + Device = load_model('config', 'Device') + device_filter = models.Q( + config__vpnclient__secret__startswith=zerotier_member_id + ) + if node.organization_id: + device_filter &= models.Q(organization_id=node.organization_id) + device = ( + Device.objects.only( + 'id', 'name', 'last_ip', 'management_ip', 'organization_id' + ) + .filter(device_filter) + .first() + ) + if not device: + return + return cls.save_device_node(device, node) + @classmethod def auto_create_netjsongraph(cls, node): if len(node.addresses) < 2: diff --git a/openwisp_network_topology/integrations/device/management/commands/__init__.py b/openwisp_network_topology/integrations/device/management/commands/__init__.py index ed55210b..36590ef4 100644 --- a/openwisp_network_topology/integrations/device/management/commands/__init__.py +++ b/openwisp_network_topology/integrations/device/management/commands/__init__.py @@ -12,6 +12,7 @@ def handle(self, *args, **kwargs): queryset = Node.objects.select_related('topology').filter( Q(topology__parser='netdiff.OpenvpnParser') | Q(topology__parser='netdiff.WireguardParser') + | Q(topology__parser='netdiff.ZeroTierParser') ) for node in queryset.iterator(): DeviceNode.auto_create(node) diff --git a/openwisp_network_topology/integrations/device/tests/test_integration.py b/openwisp_network_topology/integrations/device/tests/test_integration.py index 4caced97..f45bf99c 100644 --- a/openwisp_network_topology/integrations/device/tests/test_integration.py +++ b/openwisp_network_topology/integrations/device/tests/test_integration.py @@ -12,6 +12,7 @@ CreateConfigTemplateMixin, TestVpnX509Mixin, TestWireguardVpnMixin, + TestZeroTierVpnMixin, ) from openwisp_ipam.tests import CreateModelsMixin as SubnetIpamMixin @@ -38,6 +39,7 @@ class Base( TestVpnX509Mixin, TestWireguardVpnMixin, + TestZeroTierVpnMixin, SubnetIpamMixin, CreateConfigTemplateMixin, CreateGraphObjectsMixin, @@ -45,6 +47,8 @@ class Base( ): topology_model = Topology node_model = Node + _ZT_SERVICE_REQUESTS = 'openwisp_controller.config.api.zerotier_service.requests' + _ZT_GENERATE_IDENTITY_SUBPROCESS = 'openwisp_controller.config.base.vpn.subprocess' def _init_test_node( self, @@ -94,6 +98,24 @@ def _init_wireguard_test_node(self, topology, addresses=[], create=True, **kwarg node.save() return node + def _init_zerotier_test_node( + self, topology, addresses=None, label='test', zt_member_id=None, create=True + ): + if not addresses: + addresses = [self._TEST_ZT_MEMBER_CONFIG['address']] + node = Node( + organization=topology.organization, + topology=topology, + label=label, + addresses=addresses, + # zt peer address is `zt_memeber_id` + properties={'address': zt_member_id}, + ) + if create: + node.full_clean() + node.save() + return node + def _create_wireguard_test_env(self, parser): org = self._get_org() device, _, _ = self._create_wireguard_vpn_template() @@ -101,6 +123,33 @@ def _create_wireguard_test_env(self, parser): topology = self._create_topology(organization=org, parser=parser) return topology, device + @mock.patch(_ZT_GENERATE_IDENTITY_SUBPROCESS) + @mock.patch(_ZT_SERVICE_REQUESTS) + def _create_zerotier_test_env(self, mock_requests, mock_subprocess, parser): + mock_requests.get.side_effect = [ + # For node status + self._get_mock_response(200, response=self._TEST_ZT_NODE_CONFIG) + ] + mock_requests.post.side_effect = [ + # For create network + self._get_mock_response(200), + # For controller network join + self._get_mock_response(200), + # For controller auth and ip assignment + self._get_mock_response(200), + # For member auth and ip assignment + self._get_mock_response(200), + ] + mock_stdout = mock.MagicMock() + mock_stdout.stdout.decode.return_value = self._TEST_ZT_MEMBER_CONFIG['identity'] + mock_subprocess.run.return_value = mock_stdout + org = self._get_org() + device, _, _ = self._create_zerotier_vpn_template() + device.organization = org + topology = self._create_topology(organization=org, parser=parser) + zerotier_member_id = device.config.vpnclient_set.first().zerotier_member_id + return topology, device, zerotier_member_id + def _create_test_env(self, parser): organization = self._get_org() vpn = self._create_vpn(name='test VPN', organization=organization) @@ -198,6 +247,51 @@ def test_auto_create_wireguard(self): except KeyError: self.fail('KeyError raised') + def test_auto_create_zerotier(self): + topology, device, zerotier_member_id = self._create_zerotier_test_env( + parser='netdiff.ZeroTierParser' + ) + self.assertEqual(DeviceNode.objects.count(), 0) + with self.subTest('assert number of queries'): + with self.assertNumQueries(15): + node = self._init_zerotier_test_node( + topology, zt_member_id=zerotier_member_id + ) + self.assertEqual(DeviceNode.objects.count(), 1) + device_node = DeviceNode.objects.first() + self.assertEqual(device_node.device, device) + self.assertEqual(device_node.node, node) + + with self.subTest('not run on save'): + with mock.patch.object(transaction, 'on_commit') as on_commit: + node.save() + on_commit.assert_not_called() + + def test_auto_create_zerotier_failures(self): + topology, device, zerotier_member_id = self._create_zerotier_test_env( + parser='netdiff.ZeroTierParser' + ) + + with self.subTest('zerotier_member_id not present'): + self._init_zerotier_test_node(topology) + self.assertFalse(DeviceNode.objects.exists()) + + with self.subTest('zerotier_member_id does not exist'): + self._init_zerotier_test_node(topology, zt_member_id='non_existent_id') + self.assertFalse(DeviceNode.objects.exists()) + + with self.subTest('exception during save'): + with mock.patch.object( + DeviceNode, 'save', side_effect=Exception('test') + ) as save: + with mock.patch.object(models_logger, 'exception') as logger_exception: + self._init_zerotier_test_node( + topology, zt_member_id=zerotier_member_id + ) + save.assert_called_once() + logger_exception.assert_called_once() + self.assertEqual(DeviceNode.objects.count(), 0) + def test_filter_by_link(self): topology, device, cert = self._create_test_env(parser='netdiff.OpenvpnParser') diff --git a/openwisp_network_topology/migrations/0016_alter_topology_parser.py b/openwisp_network_topology/migrations/0016_alter_topology_parser.py index e85dc1fb..595eddf9 100644 --- a/openwisp_network_topology/migrations/0016_alter_topology_parser.py +++ b/openwisp_network_topology/migrations/0016_alter_topology_parser.py @@ -22,6 +22,7 @@ class Migration(migrations.Migration): ('netdiff.CnmlParser', 'CNML 1.0'), ('netdiff.OpenvpnParser', 'OpenVPN'), ('netdiff.WireguardParser', 'Wireguard'), + ('netdiff.ZeroTierParser', 'ZeroTier'), ], help_text='Select topology format', max_length=128, diff --git a/openwisp_network_topology/settings.py b/openwisp_network_topology/settings.py index 893a105f..4b423d4d 100644 --- a/openwisp_network_topology/settings.py +++ b/openwisp_network_topology/settings.py @@ -31,6 +31,7 @@ def get_settings_value(option, default): ('netdiff.CnmlParser', 'CNML 1.0'), ('netdiff.OpenvpnParser', 'OpenVPN'), ('netdiff.WireguardParser', 'Wireguard'), + ('netdiff.ZeroTierParser', 'ZeroTier'), ] PARSERS = DEFAULT_PARSERS + get_settings_value('PARSERS', []) diff --git a/requirements.txt b/requirements.txt index 1ea3fbff..54e763d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ openwisp-users @ https://github.com/openwisp/openwisp-users/tarball/master -netdiff @ https://github.com/openwisp/netdiff/tarball/master +netdiff @ https://github.com/openwisp/netdiff/tarball/issue-106/add-zt-parser jsonfield~=3.1.0 django-flat-json-widget @ https://github.com/openwisp/django-flat-json-widget/tarball/master openwisp-utils[celery] @ https://github.com/openwisp/openwisp-utils/tarball/master diff --git a/tests/openwisp2/sample_network_topology/migrations/0001_initial.py b/tests/openwisp2/sample_network_topology/migrations/0001_initial.py index 16caeb66..cc328597 100644 --- a/tests/openwisp2/sample_network_topology/migrations/0001_initial.py +++ b/tests/openwisp2/sample_network_topology/migrations/0001_initial.py @@ -70,6 +70,7 @@ class Migration(migrations.Migration): ('netdiff.CnmlParser', 'CNML 1.0'), ('netdiff.OpenvpnParser', 'OpenVPN'), ('netdiff.WireguardParser', 'Wireguard'), + ('netdiff.ZeroTierParser', 'ZeroTier'), ], help_text='Select topology format', max_length=128,