Skip to content

Commit

Permalink
Merge pull request #120 from gisce/NEW_cycle_csv_parser
Browse files Browse the repository at this point in the history 
New cycle csv parser
  • Loading branch information
@tinogis
tinogis authored Dec 9, 2024
2 parents 9d25d8e + fdaaf33 commit 46e1f5a
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 2 deletions.
12 changes: 10 additions & 2 deletions primestg/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@
from pytz import timezone
from primestg.ziv_service import ZivService
import base64
from primestg.cycle.cycles import CycleFile

TZ = timezone('Europe/Madrid')

from primestg.service import Service, format_timestamp
from primestg.contract_templates import CONTRACT_TEMPLATES
from primestg.utils import DLMSTemplates

import json

REPORTS = [
'get_instant_data',
Expand Down Expand Up @@ -113,7 +114,7 @@ def get_sync_sxx(**kwargs):
)
@click.option("--ip", "-i", default="10.26.0.4", help='IP i.e CNC FTPIp')
def sends_order(**kwargs):
"""Sends on of available Orders to Meter or CNC"""
"""Sends one of available Orders to Meter or CNC"""
id_pet = get_id_pet()
s = Service(id_pet, kwargs['cnc_url'], sync=True)
order_name = kwargs['order']
Expand Down Expand Up @@ -245,5 +246,12 @@ def send_ziv_cycle(**kwargs):
result = zs.send_cycle(filename=kwargs['filename'], cycle_filedata=content)
print(result.content)

@primestg.command(name='parse_cycle')
@click.argument('filename', required=True)
def parse_cycle(**kwargs):
"""Prints dict with cycle data from CNC csv"""
c = CycleFile(path=kwargs['filename'])
print(json.dumps(c.data, indent=4, default=str))

if __name__ == 'main':
primestg()
2 changes: 2 additions & 0 deletions primestg/cycle/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# coding=utf-8
from primestg.cycle.cycles import CycleFile
52 changes: 52 additions & 0 deletions primestg/cycle/cycles.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# coding=utf-8
import csv
from os.path import isfile
from os import access, R_OK
import io
import datetime
import re


class CycleFile(object):

def __init__(self, path='', content=None):
self.data = []
if path and isfile(path) and access(path, R_OK):
with io.open(path, encoding='utf-8') as fp:
self.parse(fp)
elif content is not None:
with io.StringIO(content) as fp:
self.parse(fp)

def parse(self, fp):
csvreader = csv.reader(fp, delimiter=';')
for row in csvreader:
# skip info header and tail
if len(row) < 6:
# TODO parse file date
continue
if row[0] == 'time':
# header
continue

exec_date_str, reg_name, operation, obis, class_id, element_id, result_str = row[:7]
action_return = None
if len(row) > 7:
action_return = row[-1]

exec_date = datetime.datetime.strptime(exec_date_str, '%Y/%m/%d %H:%M:%S')
clean_data = re.sub(
'array|structure', '', re.sub('[{}]', '', re.sub('[a-z_]*\{([0-9/: ]+)\}', ';\\1', result_str))
)
data = clean_data.split(';')[1:]
result = {
'timestamp': exec_date,
'reg_name': reg_name,
'operation': operation,
'obis': obis,
'class_id': class_id and int(class_id) or None,
'element_id': element_id and int(element_id) or None,
'data': data
}

self.data.append(result)
9 changes: 9 additions & 0 deletions primestg/dlms_templates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,5 +72,14 @@
{'obis': "1.0.0.4.5.255", 'class': "1", 'element': "2"}, # primary voltage
{'obis': "1.0.0.4.6.255", 'class': "1", 'element': "2"}, # secondary voltage
],
},
'GET_INSTANT': {
'description': 'Gets instant data',
'origin': 'library',
'category': 'info',
'params': [],
'data': [
{'obis': "0.0.21.0.5.255", 'class': "7", 'element': "2"}, # instant data
],
}
}
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ libComXML
pytz
python-dateutil
zeep
csv
127 changes: 127 additions & 0 deletions spec/cycle_spec.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
from primestg.cycle.cycles import CycleFile
from expects import expect, equal
import io
from datetime import datetime

from primestg.report import Report
from ast import literal_eval


with description('Parse CNC cycles'):

with before.all:

self.data_filenames = [
'spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv',
'spec/data/Ciclo_instant_data_minute_20241129_124027.csv',
'spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv',
]

self.expected_data = [
# obis 0.0.21.0.5.255 one register
[
{
'timestamp': datetime(2024, 11, 29, 11, 23, 36),
'reg_name': 'SAG0155349819',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
'data': ['2024/11/29 11:23:30', '224', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1000']
}
],
# obis 0.0.21.0.5.255 various registers
[
{
'timestamp': datetime(2024, 11, 29, 12, 40, 6),
'reg_name': 'CIR2081710470',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
# clock , v1 , i1 , v2 , i2 , v3 , i3 , sum i , inPot , outP, inR , outR, powerF
'data': ['2024/11/29 12:40:09', '235', '409', '235', '326', '236', '337', '1074', '1211', '0', '1964', '0', '520'],
},
{
'timestamp': datetime(2024, 11, 29, 12, 40, 9),
'reg_name': 'CIR0501516020',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
'data': ['2024/11/29 12:40:10', '136', '0', '142', '0', '139', '0', '0', '0', '0', '0', '0', '1000'],
},
{
'timestamp': datetime(2024, 11, 29, 12, 40, 10),
'reg_name': 'ITE0131750581',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
'data': ['2024/11/29 12:40:10', '232', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '1000'],
},
{
'timestamp': datetime(2024, 11, 29, 12, 40, 12),
'reg_name': 'SAG0186255184',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
'data': ['2024/11/29 12:40:13', '235', '0', '234', '0', '234', '0', '0', '0', '0', '0', '0', '1000']
},
{
'timestamp': datetime(2024, 11, 29, 12, 40, 21),
'reg_name': 'ZIV0034703466',
'operation': 'get',
'obis': '0.0.21.0.5.255',
'class_id': 7,
'element_id': 2,
'data': ['2024/11/29 12:40:17', '133', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0']
},
],
# Failed get
[
{
'timestamp': datetime(2024, 11, 29, 10, 23, 57),
'reg_name': 'ZIV0040318130',
'operation': '',
'obis': '',
'class_id': None,
'element_id': None,
'data': []
}
],

]

with it('Parses all files and content'):
for path in self.data_filenames:
from_path = CycleFile(path=path)

with io.open(path, encoding='utf-8') as fp:
content = fp.read()
from_text = CycleFile(content=content)

expect(from_path.data).to(equal(from_text.data))

with it('Returns expected dict'):
for index in range(0, len(self.expected_data)):
c = CycleFile(path=self.data_filenames[index])
expected_list = self.expected_data[index]

expect(len(c.data)).to(equal(len(expected_list)))
for element_index in range(0, len(expected_list)):
cycle_data = c.data[element_index]
expected = expected_list[element_index]
expect(expected['timestamp']).to(equal(cycle_data['timestamp']))
expect(expected['reg_name']).to(equal(cycle_data['reg_name']))
expect(expected['operation']).to(equal(cycle_data['operation']))
expect(expected['obis']).to(equal(cycle_data['obis']))
expect(expected['class_id']).to(equal(cycle_data['class_id']))
expect(expected['element_id']).to(equal(cycle_data['element_id']))
expect(len(expected['data'])).to(equal(len(cycle_data['data'])))
for i in range(0, len(cycle_data['data'])):
expect(expected['data'][i]).to(equal(cycle_data['data'][i]))



4 changes: 4 additions & 0 deletions spec/data/Ciclo_TAR_20TD_raw_20241129_102357_0.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
starting cycle Ciclo_TAR_20TD_raw at 2024/11/29 10:23:57
time;dev_sn;operation;obis;class_id;element_id;result;action return
2024/11/29 10:23:57;ZIV0040318130;;;;;No response received
ending cycle Ciclo_TAR_20TD_raw at 2024/11/29 10:23:57
4 changes: 4 additions & 0 deletions spec/data/Ciclo_instant_data_minute_20241129_112335_0.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
starting cycle Ciclo_instant_data_minute at 2024/11/29 11:23:35
time;dev_sn;operation;obis;class_id;element_id;result;action return
2024/11/29 11:23:36;SAG0155349819;get;0.0.21.0.5.255;7;2;array{structure{octet_string{2024/11/29 11:23:30}long_unsigned{224}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}}
ending cycle Ciclo_instant_data_minute at 2024/11/29 11:23:36
8 changes: 8 additions & 0 deletions spec/data/Ciclo_instant_data_minute_20241129_124027.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
starting cycle Ciclo_instant_data_minute at 2024/11/29 12:40:05
time;dev_sn;operation;obis;class_id;element_id;result;action return
2024/11/29 12:40:06;CIR2081710470;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:09}long_unsigned{235}long_unsigned{409}long_unsigned{235}long_unsigned{326}long_unsigned{236}long_unsigned{337}long_unsigned{1074}double_long_unsigned{1211}double_long_unsigned{0}double_long_unsigned{1964}double_long_unsigned{0}long_unsigned{520}}}
2024/11/29 12:40:09;CIR0501516020;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:10}long_unsigned{136}long_unsigned{0}long_unsigned{142}long_unsigned{0}long_unsigned{139}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}}
2024/11/29 12:40:10;ITE0131750581;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:10}long_unsigned{232}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}}
2024/11/29 12:40:12;SAG0186255184;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:13}long_unsigned{235}long_unsigned{0}long_unsigned{234}long_unsigned{0}long_unsigned{234}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{1000}}}
2024/11/29 12:40:21;ZIV0034703466;get;0.0.21.0.5.255;7;2;array{structure{date_time{2024/11/29 12:40:17}long_unsigned{133}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}double_long_unsigned{0}long_unsigned{0}}}
ending cycle Ciclo_instant_data_minute at 2024/11/29 12:40:27

0 comments on commit 46e1f5a

Please sign in to comment.