add custom field of type yearmonth (#561)

papermerge/core/alembic/README

@@ -12,3 +12,10 @@ Run migration:
  ```
  $ alembic upgrade head
  ```
+
+
+Create a migration:
+
+```
+$ alembic revision -m "add value_yearmonth column to custom_field_values"
+```

papermerge/core/alembic/versions/cea868700f4e_add_value_yearmonth_and_value_year_.py

@@ -0,0 +1,29 @@
+"""add value_yearmonth and value_year columns to custom_field_values
+
+Revision ID: cea868700f4e
+Revises: 85fda75f19f1
+Create Date: 2024-11-27 07:30:19.631965
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "cea868700f4e"
+down_revision: Union[str, None] = "85fda75f19f1"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "custom_field_values", sa.Column("value_yearmonth", sa.Float(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("custom_field_values", "value_yearmonth")

papermerge/core/constants.py

@@ -28,6 +28,8 @@
 PATH_TMPL_MOVE_DOCUMENTS = "path_tmpl_move_documents"
 # incoming (from user) date format
 INCOMING_DATE_FORMAT = "%Y-%m-%d"
+# incoming (from user) year month format
+INCOMING_YEARMONTH_FORMAT = "%Y-%m"
 
 class ContentType:
     APPLICATION_PDF = "application/pdf"

papermerge/core/features/conftest.py

@@ -412,6 +412,32 @@ def document_type_zdf(db_session: Session, user, make_custom_field):
     )
 
 
+@pytest.fixture
+def document_type_salary(db_session: Session, user, make_custom_field):
+    cf1 = make_custom_field(name="Month", type=CustomFieldType.yearmonth)
+    cf2 = make_custom_field(name="Total", type=CustomFieldType.monetary)
+    cf3 = make_custom_field(name="Company", type=CustomFieldType.date)
+
+    return dbapi.create_document_type(
+        db_session,
+        name="Salary",
+        custom_field_ids=[cf1.id, cf2.id, cf3.id],
+        user_id=user.id,
+    )
+
+
+@pytest.fixture
+def document_type_tax(db_session: Session, user, make_custom_field):
+    cf = make_custom_field(name="Year", type=CustomFieldType.int)
+
+    return dbapi.create_document_type(
+        db_session,
+        name="Tax",
+        custom_field_ids=[cf.id],
+        user_id=user.id,
+    )
+
+
 @pytest.fixture
 def make_custom_field(db_session: Session, user):
     def _make_custom_field(name: str, type: CustomFieldType):
@@ -489,3 +515,59 @@ def _make_receipt(title: str, user: orm.User, parent=None):
         return doc
 
     return _make_receipt
+
+
+@pytest.fixture
+def make_document_salary(db_session: Session, document_type_salary):
+    def _make_salary(title: str, user: orm.User, parent=None):
+        if parent is None:
+            parent_id = user.home_folder_id
+        else:
+            parent_id = parent.id
+
+        doc_id = uuid.uuid4()
+        doc = orm.Document(
+            id=doc_id,
+            ctype="document",
+            title=title,
+            user=user,
+            document_type_id=document_type_salary.id,
+            parent_id=parent_id,
+            lang="deu",
+        )
+
+        db_session.add(doc)
+
+        db_session.commit()
+
+        return doc
+
+    return _make_salary
+
+
+@pytest.fixture
+def make_document_tax(db_session: Session, document_type_tax):
+    def _make_tax(title: str, user: orm.User, parent=None):
+        if parent is None:
+            parent_id = user.home_folder_id
+        else:
+            parent_id = parent.id
+
+        doc_id = uuid.uuid4()
+        doc = orm.Document(
+            id=doc_id,
+            ctype="document",
+            title=title,
+            user=user,
+            document_type_id=document_type_tax.id,
+            parent_id=parent_id,
+            lang="deu",
+        )
+
+        db_session.add(doc)
+
+        db_session.commit()
+
+        return doc
+
+    return _make_tax

papermerge/core/features/custom_fields/db/orm.py

@@ -33,6 +33,7 @@ class CustomFieldValue(Base):
     value_int: Mapped[int] = mapped_column(nullable=True)
     value_float: Mapped[float] = mapped_column(nullable=True)
     value_monetary: Mapped[Decimal] = mapped_column(nullable=True)
+    value_yearmonth: Mapped[float] = mapped_column(nullable=True)
     created_at: Mapped[datetime] = mapped_column(insert_default=func.now())
 
     def __repr__(self):

papermerge/core/features/custom_fields/schema.py

@@ -11,6 +11,8 @@ class CustomFieldType(str, Enum):
     int = "int"
     float = "float"
     monetary = "monetary"
+    # for salaries: e.g. "February, 2023"
+    yearmonth = "yearmonth"
 
 
 class CustomField(BaseModel):

papermerge/core/features/document/db/api.py

@@ -22,7 +22,7 @@
 from papermerge.core.features.document_types.db.api import document_type_cf_count
 from papermerge.core.types import OrderEnum, CFVValueColumn
 from papermerge.core.db.common import get_ancestors
-from papermerge.core.utils.misc import str2date
+from papermerge.core.utils.misc import str2date, str2float, float2str
 from papermerge.core.pathlib import (
     abs_docver_path,
 )
@@ -64,6 +64,8 @@ def get_doc_cfv(session: Session, document_id: uuid.UUID) -> list[schema.CFV]:
     for row in session.execute(stmt):
         if row.cf_type == "date":
             value = str2date(row.cf_value)
+        elif row.cf_type == "yearmonth":
+            value = float2str(row.cf_value)
         else:
             value = row.cf_value
 
@@ -115,6 +117,8 @@ def update_doc_cfv(
             )
             if item.type.value == "date":
                 v[f"value_{item.type.value}"] = str2date(custom_fields[item.name])
+            elif item.type.value == "yearmonth":
+                v[f"value_{item.type.value}"] = str2float(custom_fields[item.name])
             else:
                 v[f"value_{item.type.value}"] = custom_fields[item.name]
             insert_values.append(v)
@@ -123,6 +127,8 @@ def update_doc_cfv(
             v = dict(id=item.custom_field_value_id)
             if item.type == "date":
                 v[f"value_{item.type.value}"] = str2date(custom_fields[item.name])
+            elif item.type.value == "yearmonth":
+                v[f"value_{item.type.value}"] = str2float(custom_fields[item.name])
             else:
                 v[f"value_{item.type.value}"] = custom_fields[item.name]
             update_values.append(v)
@@ -188,6 +194,12 @@ def get_cfv_column_name(db_session, cf_name: str) -> CFVValueColumn:
             ret = CFVValueColumn.DATE
         case "boolean":
             ret = CFVValueColumn.BOOLEAN
+        case "yearmonth":
+            ret = CFVValueColumn.YEARMONTH
+        case "int":
+            ret = CFVValueColumn.INT
+        case "float":
+            ret = CFVValueColumn.FLOAT
         case _:
             raise ValueError("Unexpected custom field type")
 

papermerge/core/features/document/db/selectors.py

@@ -73,8 +73,11 @@ def select_doc_cfv(document_id: uuid.UUID) -> Select:
         case(
             (cf.c.type == 'monetary', func.cast(cfv.value_monetary, VARCHAR)),
             (cf.c.type == 'text', func.cast(cfv.value_text, VARCHAR)),
+            (cf.c.type == 'int', func.cast(cfv.value_int, VARCHAR)),
+            (cf.c.type == 'float', func.cast(cfv.value_float, VARCHAR)),
             (cf.c.type == 'date', func.substr(func.cast(cfv.value_date, VARCHAR), 0, DATE_LEN)),
             (cf.c.type == 'boolean', func.cast(cfv.value_boolean, VARCHAR)),
+            (cf.c.type == 'yearmonth', func.cast(cfv.value_yearmonth, VARCHAR)),
         ).label("cf_value")
     ).select_from(doc).join(
         assoc,
@@ -155,6 +158,9 @@ def select_docs_by_type(
             (cf.c.type == 'date',
              func.substr(func.cast(cfv.value_date, VARCHAR), 0, DATE_LEN)),
             (cf.c.type == 'boolean', func.cast(cfv.value_boolean, VARCHAR)),
+            (cf.c.type == 'float', func.cast(cfv.value_float, VARCHAR)),
+            (cf.c.type == 'int', func.cast(cfv.value_int, VARCHAR)),
+            (cf.c.type == 'yearmonth', func.cast(cfv.value_yearmonth, VARCHAR)),
         ).label("cf_value")
     ).select_from(
         doc

papermerge/core/features/document/tests/test_dbapi_document.py

@@ -349,6 +349,78 @@ def test_get_docs_by_type_one_doc_with_nonempty_cfv(
             assert cf["Total"] is None
 
 
+def test_get_docs_by_type_one_doc_with_nonempty_cfv_with_tax_docs(
+    db_session: Session, make_document_tax, user
+):
+    """
+    `db.get_docs_by_type` must return all documents of specific type
+    regardless if they (documents) have or no associated custom field values.
+
+    In this scenario one of the returned documents has all CFVs set to
+    non empty values and the other one - to all values empty
+
+    This scenario tests documents of type "Tax" which have one custom
+    field of type 'int' (cf.name = 'Year', cf.type = 'int')
+    """
+    doc_1 = make_document_tax(title="tax_1.pdf", user=user)
+    make_document_tax(title="tax_2.pdf", user=user)
+    user_id = doc_1.user.id
+    type_id = doc_1.document_type.id
+
+    # tax_1.pdf has non-empty year values
+    dbapi.update_doc_cfv(
+        db_session,
+        document_id=doc_1.id,
+        custom_fields={"Year": 2020},
+    )
+
+    items: list[schema.DocumentCFV] = dbapi.get_docs_by_type(
+        db_session, type_id=type_id, user_id=user_id
+    )
+
+    assert len(items) == 2
+
+    # returned items are not sorted i.e. may be in any order
+    for i in range(0, 2):
+        cf = dict([(y[0], y[1]) for y in items[i].custom_fields])
+        if items[i].id == doc_1.id:
+            #  tax_1.pdf has all cf set correctly
+            assert cf["Year"] == "2020"
+        else:
+            # tax_2.pdf has all cf set to None
+            assert cf["Year"] is None
+
+
+def test_get_docs_by_type_one_tax_doc_ordered_asc(
+    db_session: Session, make_document_tax, user
+):
+    """
+    This scenario catches a bug.
+    The problem was that if you use orders by custom field of type
+    `int` (in this scenario tax document has one cf - `Year` of type `int`)
+    then there is an exception.
+    Exception should not happen.
+    """
+    doc_1 = make_document_tax(title="tax_1.pdf", user=user)
+    user_id = doc_1.user.id
+    type_id = doc_1.document_type.id
+
+    # tax_1.pdf has non-empty year values
+    dbapi.update_doc_cfv(
+        db_session,
+        document_id=doc_1.id,
+        custom_fields={"Year": 2020},
+    )
+
+    # asserts that there is no exception when ordered by
+    # (int) field "Year"
+    items: list[schema.DocumentCFV] = dbapi.get_docs_by_type(
+        db_session, type_id=type_id, user_id=user_id, order_by="Year"
+    )
+
+    assert len(items) == 1
+
+
 def test_document_version_dump(db_session, make_document, user):
     doc: schema.Document = make_document(
         title="some doc", user=user, parent=user.home_folder