Merge branch 'master' into 116-add-linear-sweep-voltammetry-to-data-t…

…ypes

.github/workflows/pylint.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -16,7 +16,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        pip install wheel setuptools pip pybind11 --upgrade
         pip install -r ./requirements/dev.txt
         pip install pylint
     - name: Analysing the code with pylint

.github/workflows/pytest.yml

@@ -7,22 +7,21 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r ./requirements/dev.txt
-        pip install pytest
-    - name: Build tests
-      run: |
-        python test_manager/__init__.py -t -g -tp
-    - name: Test the code with pytest
-      run: |
-        pytest ./test_manager/test_profiles.py
-        pytest ./test_manager/test_readers.py
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install wheel setuptools pip pybind11 --upgrade
+          pip install -r ./requirements/dev.txt
+          pip install pytest
+      - name: Build tests
+        run: |
+          python test_manager/__init__.py -t -g -tp
+      - name: Test the code with pytest
+        run: |
+          pytest .

.gitignore

@@ -33,3 +33,5 @@ test_manager/test_readers.py
 /test_manager/test_profiles.py
 /test_files/
 /.venv/
+logs/
+

converter_app/converters.py

@@ -1,4 +1,5 @@
 import copy
+import datetime
 import logging
 import os
 import re
@@ -364,15 +365,21 @@ def match_profile(cls, client_id, file_data):
         """
         converter = None
         matches = 0
-
+        latest_profile_uploaded = 0
         for profile in Profile.list(client_id):
             current_converter = cls(profile, file_data)
             current_matches = current_converter.match()
-
+            try:
+                profile_uploaded = datetime.datetime.fromisoformat(
+                    profile.as_dict['data']['metadata'].get('uploaded')).timestamp()
+            except (ValueError, TypeError):
+                profile_uploaded = 1
             logger.info('profile=%s matches=%s', profile.id, current_matches)
-
-            if current_matches is not False and current_matches > matches:
+            if (current_matches is not False and
+                    (current_matches > matches or current_matches == matches and
+                    profile_uploaded > latest_profile_uploaded)):
+                matches = max(matches, current_matches)
+                latest_profile_uploaded = profile_uploaded
                 converter = current_converter
-                matches = current_matches
 
         return converter

converter_app/models.py

@@ -163,7 +163,7 @@ def list(cls, client_id):
         profiles_path = Path(current_app.config['PROFILES_DIR']).joinpath(client_id)
 
         if profiles_path.exists():
-            for file_path in Path.iterdir(profiles_path):
+            for file_path in sorted(Path.iterdir(profiles_path)):
                 profile_id = str(file_path.with_suffix('').name)
                 profile_data = cls.load(file_path)
                 yield cls(profile_data, client_id, profile_id)

converter_app/readers/afm.py

@@ -0,0 +1,57 @@
+import logging
+import re
+
+from converter_app.readers.helper.reader import Readers
+from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix
+
+logger = logging.getLogger(__name__)
+
+
+class AfmReader(Reader):
+    """
+    Implementation of the Afm Reader. It extends converter_app.readers.helper.base.Reader
+    """
+    identifier = 'afm_reader'
+    priority = 10
+
+    def check(self):
+        return self.file.suffix.lower() == '.spm'
+
+    def prepare_tables(self):
+        tables = []
+        table = self.append_table(tables)
+        header = False
+        lines = []
+
+        for line in self.file.fp.readlines():
+            lines.append(line)
+            try:
+                row = line.decode('utf-8').strip()
+            except UnicodeDecodeError:
+                continue
+
+            if re.search(r'[^-<|>!?+*:.,#@%&~_"\'/\\a-zA-Z0-9\[\](){}\s]', row):
+                continue
+
+            if len(row) > 1 and row[1] == '*':
+                header = True
+
+            if header:
+                table['header'].append(row[2:])
+                header = False
+            else:
+                k_v = [x.strip() for x in row[1:].split(':', 1)]
+                table['metadata'].add_unique(k_v[0], k_v[-1])
+
+        extracted_table = self.append_table(tables)
+        for term in search_terms_matrix:
+            value = table['metadata'].get(term[2]) if table['metadata'].get(term[2]) is not None else ''
+            extracted_table['metadata'].add_unique(term[0], value)
+
+        extracted_table['metadata'] = convert_units(extracted_table['metadata'], 2)
+
+        return tables
+
+
+Readers.instance().register(AfmReader)

converter_app/readers/asc_zip.py

@@ -15,7 +15,10 @@ class AscZipReader(Reader):
     """
     identifier = 'asc_zip_reader'
     priority = 10
-    filedata = {}
+
+    def __init__(self, file):
+        super().__init__(file)
+        self.filedata = {}
 
     # two or more chars in row
 

converter_app/readers/ascii.py

@@ -46,7 +46,7 @@ def prepare_tables(self):
                 row = row.replace('n.a.', '')
                 float_match = self.float_pattern.findall(row)
                 if float_match:
-                    float_match = [self.get_value(float_str) for float_str in float_match]
+                    float_match = [self.get_value(float_str.strip()) for float_str in float_match]
                     count = len(float_match)
 
                     if table['rows'] and count != previous_count:

converter_app/readers/csm.py

@@ -0,0 +1,34 @@
+import logging
+
+from converter_app.readers.helper.reader import Readers
+from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix
+from converter_app.readers.html import HtmlReader
+
+logger = logging.getLogger(__name__)
+
+
+class CsmReader(HtmlReader):
+    """
+        Implementation of the Csm Reader. It extends converter_app.readers.helper.base.Reader
+    """
+    identifier = 'csm_reader'
+    priority = 10
+
+    def check(self):
+        is_check = super().check()
+        if is_check:
+            return '<div id="metainfo">' in self.file.string
+
+    def prepare_tables(self):
+        tables = super().prepare_tables()
+        table = self.append_table(tables)
+        for term in search_terms_matrix:
+            for key, val in tables[-2]['metadata'].items():
+                if key == term[1]:
+                    table['metadata'].add_unique(term[0], val)
+        table = self.append_table(tables)
+        table['metadata'] = convert_units(tables[-1]['metadata'] | tables[-2]['metadata'], 1)
+        return tables
+
+
+Readers.instance().register(CsmReader)

converter_app/readers/excel.py

@@ -4,7 +4,7 @@
 import openpyxl
 
 from converter_app.readers.helper.reader import Readers
-from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.base import Reader, Table
 
 logger = logging.getLogger(__name__)
 
@@ -19,6 +19,8 @@ class ExcelReader(Reader):
     def __init__(self, file):
         super().__init__(file)
         self.wb = None
+        self._table_row_meta = Table()
+        self._table_col_meta = Table()
 
     def check(self):
         """
@@ -40,29 +42,35 @@ def check(self):
 
     def prepare_tables(self):
         tables = []
-
+        # A, B, C
+        # C , C ,C ,C
         # loop over worksheets
         for ws in self.wb:
             self.append_table(tables)
+            keys = []
 
             previous_shape = None
             for row in ws.values:
                 shape = self.get_shape(row)
 
                 if 's' in shape:
                     # there is a string in this row, this cant be the table
-
+                    self._set_col_metadata(row)
                     if tables[-1]['rows']:
                         # if a table is already there, this must be a new header
                         self.append_table(tables)
+                        keys = []
 
                     tables[-1]['header'].append('\t'.join([str(cell) for cell in row]))
+                    self._set_row_metadata(keys, row, True)
 
                 elif 'f' in shape:
                     if tables[-1]['rows'] and shape != previous_shape:
                         # start a new table if the shape has changed
                         self.append_table(tables)
-
+                        keys = []
+                    else:
+                        self._set_row_metadata(keys, row, False)
                     # this row has floats but no strings, this is the "real" table
                     values = [row[i] for i, value in enumerate(shape) if value == 'f']
                     tables[-1]['rows'].append(values)
@@ -73,7 +81,29 @@ def prepare_tables(self):
 
                 # store shape and row for the next iteration
                 previous_shape = shape
-
+        tables.append(self._table_row_meta)
+        tables.append(self._table_col_meta)
         return tables
 
+    def _set_row_metadata(self, keys, row, set_keys):
+        """Sets the metadata for the table."""
+        if len(keys) == 0:
+            if set_keys:
+                for cell in row:
+                    keys.append(str(cell))
+        else:
+            for cell in row:
+                key = keys.pop(0)
+                if set_keys:
+                    keys.append(str(cell))
+                if key != 'None':
+                    self._table_row_meta['metadata'].add_unique(key, str(cell))
+
+    def _set_col_metadata(self, row):
+        row = [x for x in row if x != 'None']
+        shape = self.get_shape(row)
+        if len(shape) < 4 and shape[0] == 's':
+            for val in row[1:]:
+                self._table_col_meta['metadata'].add_unique(row[0], str(val))
+
 Readers.instance().register(ExcelReader)

converter_app/readers/helper/base.py

@@ -19,6 +19,15 @@ def __init__(self):
             'rows': []
         })
 
+    def add_metadata(self, key, value):
+        """
+        Add metadata to table
+        :param key: Key of the metadata
+        :param value: Value of the metadata
+        :return:
+        """
+        self['metadata'].add_unique(key, value)
+
     def __add__(self, other):
         raise NotImplementedError
 
@@ -47,7 +56,7 @@ class Reader:
     """
     Base reader. Any reader needs to extend this abstract reader.
     """
-    float_pattern = re.compile(r'(-?\d+[,.]*\d*[eE+\-\d]*)\S*')
+    float_pattern = re.compile(r'[-+]?[0-9]*[.,]?[0-9]+(?:[eE][-+]?[0-9]+)?\s*')
     float_de_pattern = re.compile(r'(-?[\d.]+,\d*[eE+\-\d]*)')
     float_us_pattern = re.compile(r'(-?[\d,]+.\d*[eE+\-\d]*)')
 
@@ -117,7 +126,7 @@ def get_tables(self) -> list[Table]:
                         'name': f'Column #{idx + start_len_c}'
                     } for idx, value in enumerate(table['rows'][0][start_len_c:])]
                     table['columns'] = sorted(table['columns'], key=lambda x: int(x['key']))
-                for k,v in enumerate(table['columns'][:should_len_c]):
+                for k, v in enumerate(table['columns'][:should_len_c]):
                     v['key'] = f'{k}'
 
             table['metadata']['rows'] = str(len(table['rows']))
@@ -165,16 +174,31 @@ def get_shape(self, row) -> list:
             if cell is None:
                 shape.append(None)
             else:
+                if isinstance(cell, datetime):
+                    shape.append('f')
+                    continue
                 cell = str(cell).strip()
                 if cell in self.empty_values:
                     shape.append('')
-                elif self.float_pattern.match(cell):
+                elif self.float_pattern.fullmatch(cell):
                     shape.append('f')
                 else:
                     shape.append('s')
 
         return shape
 
+    def as_number(self, value: str) -> float | int:
+        """
+        Returns a numeric value if possible:
+
+        :raises ValueError: If not convertable
+        :param value: as string
+        :return: numeric value either int or float
+        """
+        if re.match(r'^[+-]?\d+$', value) is not None:
+            return int(value)
+        return float(self.get_value(value))
+
     def get_value(self, value: str) -> str:
         """
         Checks if values is a stringified float and makes it to a standard.