From 0e7dbb36925a3e9c26d40175da247be83752c40a Mon Sep 17 00:00:00 2001 From: tinogis Date: Wed, 4 Dec 2024 16:35:59 +0100 Subject: [PATCH 1/5] NEW cycle csv result parser --- primestg/cycle/__init__.py | 2 ++ primestg/cycle/cycles.py | 52 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 55 insertions(+) create mode 100644 primestg/cycle/__init__.py create mode 100644 primestg/cycle/cycles.py diff --git a/primestg/cycle/__init__.py b/primestg/cycle/__init__.py new file mode 100644 index 0000000..35c0889 --- /dev/null +++ b/primestg/cycle/__init__.py @@ -0,0 +1,2 @@ +# coding=utf-8 +from primestg.cycle.cycles import CycleFile \ No newline at end of file diff --git a/primestg/cycle/cycles.py b/primestg/cycle/cycles.py new file mode 100644 index 0000000..7c9c9a8 --- /dev/null +++ b/primestg/cycle/cycles.py @@ -0,0 +1,52 @@ +# coding=utf-8 +import csv +from os.path import isfile +from os import access, R_OK +import io +import datetime +import re + + +class CycleFile(object): + + def __init__(self, path='', content=None): + self.data = [] + if path and isfile(path) and access(path, R_OK): + with io.open(path, encoding='utf-8') as fp: + self.parse(fp) + elif content is not None: + with io.StringIO(content) as fp: + self.parse(fp) + + def parse(self, fp): + csvreader = csv.reader(fp, delimiter=';') + for row in csvreader: + # skip info header and tail + if len(row) < 6: + # TODO parse file date + continue + if row[0] == 'time': + # header + continue + + exec_date_str, reg_name, operation, obis, class_id, element_id, result_str = row[:7] + action_return = None + if len(row) > 7: + action_return = row[-1] + + exec_date = datetime.datetime.strptime(exec_date_str, '%Y/%m/%d %H:%M:%S') + clean_data = re.sub( + 'array|structure', '', re.sub('[{}]', '', re.sub('[a-z_]*\{([0-9/: ]+)\}', ';\\1', result_str)) + ) + data = clean_data.split(';')[1:] + result = { + 'timestamp': exec_date, + 'reg_name': reg_name, + 'operation': operation, + 'obis': obis, + 'class_id': class_id and int(class_id) or None, + 'element_id': element_id and int(element_id) or None, + 'data': data + } + + self.data.append(result) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 61a5abf..f3731cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ libComXML pytz python-dateutil zeep +csv From a3671c1742f99c627df28e1c997d707d5ef3b15e Mon Sep 17 00:00:00 2001 From: tinogis Date: Wed, 4 Dec 2024 16:36:46 +0100 Subject: [PATCH 2/5] TEST cycle csv result parser --- spec/cycle_spec.py | 127 ++++++++++++++++++ .../Ciclo_TAR_20TD_raw_20241129_102357_0.csv | 4 + ..._instant_data_minute_20241129_112335_0.csv | 4 + ...lo_instant_data_minute_20241129_124027.csv | 8 ++ 4 files changed, 143 insertions(+) create mode 100644 spec/cycle_spec.py create mode 100644 spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv create mode 100644 spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv create mode 100644 spec/data/Ciclo_instant_data_minute_20241129_124027.csv diff --git a/spec/cycle_spec.py b/spec/cycle_spec.py new file mode 100644 index 0000000..e4d8d72 --- /dev/null +++ b/spec/cycle_spec.py @@ -0,0 +1,127 @@ +from primestg.cycle.cycles import CycleFile +from expects import expect, equal +import io +from datetime import datetime + +from primestg.report import Report +from ast import literal_eval + + +with description('Parse CNC cycles'): + + with before.all: + + self.data_filenames = [ + 'spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv', + 'spec/data/Ciclo_instant_data_minute_20241129_124027.csv', + 'spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv', + ] + + self.expected_data = [ + # obis 0.0.21.0.5.255 one register + [ + { + 'timestamp': datetime(2024, 11, 29, 11, 23, 36), + 'reg_name': 'SAG0155349819', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + 'data': ['2024/11/29 11:23:30', '224', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1000'] + } + ], + # obis 0.0.21.0.5.255 various registers + [ + { + 'timestamp': datetime(2024, 11, 29, 12, 40, 6), + 'reg_name': 'CIR2081710470', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + # clock , v1 , i1 , v2 , i2 , v3 , i3 , sum i , inPot , outP, inR , outR, powerF + 'data': ['2024/11/29 12:40:09', '235', '409', '235', '326', '236', '337', '1074', '1211', '0', '1964', '0', '520'], + }, + { + 'timestamp': datetime(2024, 11, 29, 12, 40, 9), + 'reg_name': 'CIR0501516020', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + 'data': ['2024/11/29 12:40:10', '136', '0', '142', '0', '139', '0', '0', '0', '0', '0', '0', '1000'], + }, + { + 'timestamp': datetime(2024, 11, 29, 12, 40, 10), + 'reg_name': 'ITE0131750581', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + 'data': ['2024/11/29 12:40:10', '232', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1000'], + }, + { + 'timestamp': datetime(2024, 11, 29, 12, 40, 12), + 'reg_name': 'SAG0186255184', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + 'data': ['2024/11/29 12:40:13', '235', '0', '234', '0', '234', '0', '0', '0', '0', '0', '0', '1000'] + }, + { + 'timestamp': datetime(2024, 11, 29, 12, 40, 21), + 'reg_name': 'ZIV0034703466', + 'operation': 'get', + 'obis': '0.0.21.0.5.255', + 'class_id': 7, + 'element_id': 2, + 'data': ['2024/11/29 12:40:17', '133', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'] + }, + ], + # Failed get + [ + { + 'timestamp': datetime(2024, 11, 29, 10, 23, 57), + 'reg_name': 'ZIV0040318130', + 'operation': '', + 'obis': '', + 'class_id': None, + 'element_id': None, + 'data': [] + } + ], + + ] + + with it('Parses all files and content'): + for path in self.data_filenames: + from_path = CycleFile(path=path) + + with io.open(path, encoding='utf-8') as fp: + content = fp.read() + from_text = CycleFile(content=content) + + expect(from_path.data).to(equal(from_text.data)) + + with it('Returns expected dict'): + for index in range(0, len(self.expected_data)): + c = CycleFile(path=self.data_filenames[index]) + expected_list = self.expected_data[index] + + expect(len(c.data)).to(equal(len(expected_list))) + for element_index in range(0, len(expected_list)): + cycle_data = c.data[element_index] + expected = expected_list[element_index] + expect(expected['timestamp']).to(equal(cycle_data['timestamp'])) + expect(expected['reg_name']).to(equal(cycle_data['reg_name'])) + expect(expected['operation']).to(equal(cycle_data['operation'])) + expect(expected['obis']).to(equal(cycle_data['obis'])) + expect(expected['class_id']).to(equal(cycle_data['class_id'])) + expect(expected['element_id']).to(equal(cycle_data['element_id'])) + expect(len(expected['data'])).to(equal(len(cycle_data['data']))) + for i in range(0, len(cycle_data['data'])): + expect(expected['data'][i]).to(equal(cycle_data['data'][i])) + + + diff --git a/spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv b/spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv new file mode 100644 index 0000000..192aa5c --- /dev/null +++ b/spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv @@ -0,0 +1,4 @@ +starting cycle Ciclo_TAR_20TD_raw at 2024/11/29 10:23:57 +time;dev_sn;operation;obis;class_id;element_id;result;action return +2024/11/29 10:23:57;ZIV0040318130;;;;;No response received +ending cycle Ciclo_TAR_20TD_raw at 2024/11/29 10:23:57 diff --git a/spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv b/spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv new file mode 100644 index 0000000..26d99b7 --- /dev/null +++ b/spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv @@ -0,0 +1,4 @@ +starting cycle Ciclo_instant_data_minute at 2024/11/29 11:23:35 +time;dev_sn;operation;obis;class_id;element_id;result;action return +2024/11/29 11:23:36;SAG0155349819;get;0.0.21.0.5.255;7;2;array{structure{octet_string{2024/11/29 11:23:30}long_unsigned{224}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}} +ending cycle Ciclo_instant_data_minute at 2024/11/29 11:23:36 diff --git a/spec/data/Ciclo_instant_data_minute_20241129_124027.csv b/spec/data/Ciclo_instant_data_minute_20241129_124027.csv new file mode 100644 index 0000000..d33374b --- /dev/null +++ b/spec/data/Ciclo_instant_data_minute_20241129_124027.csv @@ -0,0 +1,8 @@ +starting cycle Ciclo_instant_data_minute at 2024/11/29 12:40:05 +time;dev_sn;operation;obis;class_id;element_id;result;action return +2024/11/29 12:40:06;CIR2081710470;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:09}long_unsigned{235}long_unsigned{409}long_unsigned{235}long_unsigned{326}long_unsigned{236}long_unsigned{337}long_unsigned{1074}double_long_unsigned{1211}double_long_unsigned{0}double_long_unsigned{1964}double_long_unsigned{0}long_unsigned{520}}} +2024/11/29 12:40:09;CIR0501516020;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:10}long_unsigned{136}long_unsigned{0}long_unsigned{142}long_unsigned{0}long_unsigned{139}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}} +2024/11/29 12:40:10;ITE0131750581;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:10}long_unsigned{232}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}} +2024/11/29 12:40:12;SAG0186255184;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:13}long_unsigned{235}long_unsigned{0}long_unsigned{234}long_unsigned{0}long_unsigned{234}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}} +2024/11/29 12:40:21;ZIV0034703466;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:17}long_unsigned{133}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{0}}} +ending cycle Ciclo_instant_data_minute at 2024/11/29 12:40:27 From 2f75368bc1020761d7b7b9a36b0cd726e8418aef Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 5 Dec 2024 09:43:14 +0100 Subject: [PATCH 3/5] NEW add parse_cycle cli option --- primestg/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/primestg/cli.py b/primestg/cli.py index 68655d6..82a7553 100644 --- a/primestg/cli.py +++ b/primestg/cli.py @@ -6,13 +6,14 @@ from pytz import timezone from primestg.ziv_service import ZivService import base64 +from primestg.cycle.cycles import CycleFile TZ = timezone('Europe/Madrid') from primestg.service import Service, format_timestamp from primestg.contract_templates import CONTRACT_TEMPLATES from primestg.utils import DLMSTemplates - +import json REPORTS = [ 'get_instant_data', @@ -245,5 +246,12 @@ def send_ziv_cycle(**kwargs): result = zs.send_cycle(filename=kwargs['filename'], cycle_filedata=content) print(result.content) +@primestg.command(name='parse_cycle') +@click.argument('filename', required=True) +def parse_cycle(**kwargs): + """Prints dict with cycle data from CNC csv""" + c = CycleFile(path=kwargs['filename']) + print(json.dumps(c.data, indent=4, default=str)) + if __name__ == 'main': primestg() From caaeb9b3be064b33445df2406e2919ca821dd65a Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 5 Dec 2024 09:45:20 +0100 Subject: [PATCH 4/5] NEW GET_INSTANT DLMS template --- primestg/dlms_templates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/primestg/dlms_templates.py b/primestg/dlms_templates.py index e253573..d776c5a 100644 --- a/primestg/dlms_templates.py +++ b/primestg/dlms_templates.py @@ -72,5 +72,14 @@ {'obis': "1.0.0.4.5.255", 'class': "1", 'element': "2"}, # primary voltage {'obis': "1.0.0.4.6.255", 'class': "1", 'element': "2"}, # secondary voltage ], + }, + 'GET_INSTANT': { + 'description': 'Gets instant data', + 'origin': 'library', + 'category': 'info', + 'params': [], + 'data': [ + {'obis': "0.0.21.0.5.255", 'class': "7", 'element': "2"}, # instant data + ], } } From fdaaf33a412882d3f2200aa503af7a54dd3d145f Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 5 Dec 2024 09:46:44 +0100 Subject: [PATCH 5/5] typo error --- primestg/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primestg/cli.py b/primestg/cli.py index 82a7553..20b74eb 100644 --- a/primestg/cli.py +++ b/primestg/cli.py @@ -114,7 +114,7 @@ def get_sync_sxx(**kwargs): ) @click.option("--ip", "-i", default="10.26.0.4", help='IP i.e CNC FTPIp') def sends_order(**kwargs): - """Sends on of available Orders to Meter or CNC""" + """Sends one of available Orders to Meter or CNC""" id_pet = get_id_pet() s = Service(id_pet, kwargs['cnc_url'], sync=True) order_name = kwargs['order']