Merge branch 'develop' into records_search_rank

docs/docs/api/rpc.md

@@ -196,7 +196,10 @@ To use an RPC function:
       members:
       - list_
       - delete
+      - run
       - ExplorationInfo
+      - ExplorationDef
+      - ExplorationResult
 
 ## Roles
 

mathesar/migrations/0014_remove_columnmetadata_duration_show_units_and_more.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.11 on 2024-07-31 12:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mathesar', '0013_alter_columnmetadata_bool_false_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='columnmetadata',
+            name='duration_show_units',
+        ),
+        migrations.RemoveField(
+            model_name='columnmetadata',
+            name='num_show_as_perc',
+        ),
+        migrations.AddField(
+            model_name='columnmetadata',
+            name='num_format',
+            field=models.CharField(choices=[('english', 'english'), ('german', 'german'), ('french', 'french'), ('hindi', 'hindi'), ('swiss', 'swiss')], null=True),
+        ),
+        migrations.AddField(
+            model_name='columnmetadata',
+            name='num_grouping',
+            field=models.CharField(choices=[('always', 'always'), ('auto', 'auto'), ('never', 'never')], null=True),
+        ),
+    ]

mathesar/models/base.py

@@ -94,7 +94,14 @@ class ColumnMetaData(BaseModel):
     bool_false = models.CharField(null=True)
     num_min_frac_digits = models.PositiveIntegerField(null=True)
     num_max_frac_digits = models.PositiveIntegerField(null=True)
-    num_show_as_perc = models.BooleanField(null=True)
+    num_grouping = models.CharField(
+        choices=[("always", "always"), ("auto", "auto"), ("never", "never")],
+        null=True
+    )
+    num_format = models.CharField(
+        choices=[("english", "english"), ("german", "german"), ("french", "french"), ("hindi", "hindi"), ("swiss", "swiss")],
+        null=True
+    )
     mon_currency_symbol = models.CharField(null=True)
     mon_currency_location = models.CharField(
         choices=[("after-minus", "after-minus"), ("end-with-space", "end-with-space")],
@@ -104,7 +111,6 @@ class ColumnMetaData(BaseModel):
     date_format = models.CharField(null=True)
     duration_min = models.CharField(max_length=255, null=True)
     duration_max = models.CharField(max_length=255, null=True)
-    duration_show_units = models.BooleanField(null=True)
 
     class Meta:
         constraints = [

mathesar/rpc/columns/metadata.py

@@ -25,14 +25,14 @@ class ColumnMetaDataRecord(TypedDict):
         bool_false: A string to display for `false` values.
         num_min_frac_digits: Minimum digits shown after the decimal point.
         num_max_frac_digits: Maximum digits shown after the decimal point.
-        num_show_as_perc: Whether to show a numeric value as a percentage.
+        num_grouping: Specifies how grouping separators are displayed for numeric values.
+        num_format: Specifies the locale-specific format for displaying numeric values.
         mon_currency_symbol: The currency symbol shown for money value.
         mon_currency_location: Where the currency symbol should be shown.
         time_format: A string representing the format of time values.
         date_format: A string representing the format of date values.
         duration_min: The smallest unit for displaying durations.
         duration_max: The largest unit for displaying durations.
-        duration_show_units: Whether to show the units for durations.
     """
     database_id: int
     table_oid: int
@@ -42,14 +42,14 @@ class ColumnMetaDataRecord(TypedDict):
     bool_false: Optional[str]
     num_min_frac_digits: Optional[int]
     num_max_frac_digits: Optional[int]
-    num_show_as_perc: Optional[bool]
+    num_grouping: Optional[str]
+    num_format: Optional[str]
     mon_currency_symbol: Optional[str]
     mon_currency_location: Optional[Literal["after-minus", "end-with-space"]]
     time_format: Optional[str]
     date_format: Optional[str]
     duration_min: Optional[str]
     duration_max: Optional[str]
-    duration_show_units: Optional[bool]
 
     @classmethod
     def from_model(cls, model):
@@ -62,14 +62,14 @@ def from_model(cls, model):
             bool_false=model.bool_false,
             num_min_frac_digits=model.num_min_frac_digits,
             num_max_frac_digits=model.num_max_frac_digits,
-            num_show_as_perc=model.num_show_as_perc,
+            num_grouping=model.num_grouping,
+            num_format=model.num_format,
             mon_currency_symbol=model.mon_currency_symbol,
             mon_currency_location=model.mon_currency_location,
             time_format=model.time_format,
             date_format=model.date_format,
             duration_min=model.duration_min,
             duration_max=model.duration_max,
-            duration_show_units=model.duration_show_units,
         )
 
 
@@ -84,29 +84,29 @@ class ColumnMetaDataBlob(TypedDict):
         bool_false: A string to display for `false` values.
         num_min_frac_digits: Minimum digits shown after the decimal point.
         num_max_frac_digits: Maximum digits shown after the decimal point.
-        num_show_as_perc: Whether to show a numeric value as a percentage.
+        num_grouping: Specifies how grouping separators are displayed for numeric values.
+        num_format: Specifies the locale-specific format for displaying numeric values.
         mon_currency_symbol: The currency symbol shown for money value.
         mon_currency_location: Where the currency symbol should be shown.
         time_format: A string representing the format of time values.
         date_format: A string representing the format of date values.
         duration_min: The smallest unit for displaying durations.
         duration_max: The largest unit for displaying durations.
-        duration_show_units: Whether to show the units for durations.
     """
     attnum: int
     bool_input: Optional[Literal["dropdown", "checkbox"]]
     bool_true: Optional[str]
     bool_false: Optional[str]
     num_min_frac_digits: Optional[int]
     num_max_frac_digits: Optional[int]
-    num_show_as_perc: Optional[bool]
+    num_grouping: Optional[str]
+    num_format: Optional[str]
     mon_currency_symbol: Optional[str]
     mon_currency_location: Optional[Literal["after-minus", "end-with-space"]]
     time_format: Optional[str]
     date_format: Optional[str]
     duration_min: Optional[str]
     duration_max: Optional[str]
-    duration_show_units: Optional[bool]
 
     @classmethod
     def from_model(cls, model):
@@ -117,14 +117,14 @@ def from_model(cls, model):
             bool_false=model.bool_false,
             num_min_frac_digits=model.num_min_frac_digits,
             num_max_frac_digits=model.num_max_frac_digits,
-            num_show_as_perc=model.num_show_as_perc,
+            num_grouping=model.num_grouping,
+            num_format=model.num_format,
             mon_currency_symbol=model.mon_currency_symbol,
             mon_currency_location=model.mon_currency_location,
             time_format=model.time_format,
             date_format=model.date_format,
             duration_min=model.duration_min,
             duration_max=model.duration_max,
-            duration_show_units=model.duration_show_units
         )
 
 

mathesar/rpc/explorations.py

@@ -3,16 +3,17 @@
 """
 from typing import Optional, TypedDict
 
-from modernrpc.core import rpc_method
+from modernrpc.core import rpc_method, REQUEST_KEY
 from modernrpc.auth.basic import http_basic_auth_login_required
 
 from mathesar.rpc.exceptions.handlers import handle_rpc_exceptions
-from mathesar.utils.explorations import get_explorations, delete_exploration
+from mathesar.rpc.utils import connect
+from mathesar.utils.explorations import get_explorations, delete_exploration, run_exploration
 
 
 class ExplorationInfo(TypedDict):
     """
-    Information about a Exploration.
+    Information about an exploration.
 
     Attributes:
         id: The Django id of an exploration.
@@ -50,6 +51,94 @@ def from_model(cls, model):
         )
 
 
+class ExplorationDef(TypedDict):
+    """
+    Definition about a runnable exploration.
+
+    Attributes:
+        base_table_oid: The OID of the base table of the exploration on the database.
+        initial_columns: A list describing the columns to be included in the exploration.
+        display_names: A map between the actual column names on the database and the alias to be displayed.
+        transformations: A list describing the transformations to be made on the included columns.
+        limit: Specifies the number of rows to return.(default 100)
+        offset: Specifies the number of rows to skip.(default 0)
+        filter: A dict describing filters to be applied to an exploration.
+            e.g. Here is a dict describing getting records from exploration where "col1" = NULL and "col2" = "abc"
+            ```
+            {"and": [
+                {"null": [
+                    {"column_name": ["col1"]},
+                ]},
+                {"equal": [
+                    {"to_lowercase": [
+                        {"column_name": ["col2"]},
+                    ]},
+                    {"literal": ["abc"]},
+                ]},
+            ]}
+            ```
+            Refer to db/functions/base.py for all the possible filters.
+        order_by: A list of dicts, where each dict has a `field` and `direction` field.
+                  Here the value for `field` should be column name and `direction` should be either `asc` or `desc`.
+        search: A list of dicts, where each dict has a `column` and `literal` field.
+                Here the value for `column` should be a column name and `literal` should be a string to be searched in the aforementioned column.
+        duplicate_only: A list of column names for which you want duplicate records.
+    """
+    base_table_oid: int
+    initial_columns: list
+    display_names: dict
+    transformations: Optional[list]
+    limit: Optional[int]
+    offset: Optional[int]
+    filter: Optional[dict]
+    order_by: Optional[list[dict]]
+    search: Optional[list[dict]]
+    duplicate_only: Optional[list]
+
+
+class ExplorationResult(TypedDict):
+    """
+    Result of an exploration run.
+
+    Attributes:
+        query: A dict describing the exploration that ran.
+        records: A dict describing the total count of records along with the contents of those records.
+        output_columns: A tuple describing the names of the columns included in the exploration.
+        column_metadata: A dict describing the metadata applied to included columns.
+        limit: Specifies the max number of rows returned.(default 100)
+        offset: Specifies the number of rows skipped.(default 0)
+        filter: A dict describing filters applied to an exploration.
+        order_by: The ordering applied to the columns of an exploration.
+        search: Specifies a list of dicts containing column names and searched expression.
+        duplicate_only: A list of column names for which you want duplicate records.
+    """
+    query: dict
+    records: dict
+    output_columns: tuple
+    column_metadata: dict
+    limit: Optional[int]
+    offset: Optional[int]
+    filter: Optional[dict]
+    order_by: Optional[list[dict]]
+    search: Optional[list[dict]]
+    duplicate_only: Optional[list]
+
+    @classmethod
+    def from_dict(cls, e):
+        return cls(
+            query=e["query"],
+            records=e["records"],
+            output_columns=e["output_columns"],
+            column_metadata=e["column_metadata"],
+            limit=e["limit"],
+            offset=e["offset"],
+            filter=e["filter"],
+            order_by=e["order_by"],
+            search=e["search"],
+            duplicate_only=e["duplicate_only"]
+        )
+
+
 @rpc_method(name="explorations.list")
 @http_basic_auth_login_required
 @handle_rpc_exceptions
@@ -78,3 +167,20 @@ def delete(*, exploration_id: int, **kwargs) -> None:
         exploration_id: The Django id of the exploration to delete.
     """
     delete_exploration(exploration_id)
+
+
+@rpc_method(name="explorations.run")
+@http_basic_auth_login_required
+@handle_rpc_exceptions
+def run(*, exploration_def: ExplorationDef, database_id: int, **kwargs) -> ExplorationResult:
+    """
+    Run an exploration.
+
+    Args:
+        exploration_def: A dict describing an exploration to run.
+        database_id: The Django id of the database containing the base table for the exploration.
+    """
+    user = kwargs.get(REQUEST_KEY).user
+    with connect(database_id, user) as conn:
+        exploration_result = run_exploration(exploration_def, database_id, conn)
+    return ExplorationResult.from_dict(exploration_result)

mathesar/tests/rpc/columns/test_c_metadata.py

@@ -20,18 +20,18 @@ def mock_get_columns_meta_data(_table_oid, _database_id):
             ColumnMetaData(
                 database=db_model, table_oid=_table_oid, attnum=2,
                 bool_input="dropdown", bool_true="TRUE", bool_false="FALSE",
-                num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False,
+                num_min_frac_digits=5, num_max_frac_digits=10, num_grouping="force-yes",
                 mon_currency_symbol="EUR", mon_currency_location="end-with-space",
                 time_format=None, date_format=None,
-                duration_min=None, duration_max=None, duration_show_units=True,
+                duration_min=None, duration_max=None, num_format="english",
             ),
             ColumnMetaData(
                 database=db_model, table_oid=_table_oid, attnum=8,
                 bool_input="checkbox", bool_true="true", bool_false="false",
-                num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True,
+                num_min_frac_digits=2, num_max_frac_digits=8, num_grouping="force-no",
                 mon_currency_symbol="$", mon_currency_location="after-minus",
                 time_format=None, date_format=None,
-                duration_min=None, duration_max=None, duration_show_units=True,
+                duration_min=None, duration_max=None, num_format="german",
             )
         ]
 
@@ -41,18 +41,18 @@ def mock_get_columns_meta_data(_table_oid, _database_id):
         metadata.ColumnMetaDataRecord(
             database_id=database_id, table_oid=table_oid, attnum=2,
             bool_input="dropdown", bool_true="TRUE", bool_false="FALSE",
-            num_min_frac_digits=5, num_max_frac_digits=10, num_show_as_perc=False,
+            num_min_frac_digits=5, num_max_frac_digits=10, num_grouping="force-yes",
             mon_currency_symbol="EUR", mon_currency_location="end-with-space",
             time_format=None, date_format=None,
-            duration_min=None, duration_max=None, duration_show_units=True,
+            duration_min=None, duration_max=None, num_format="english",
         ),
         metadata.ColumnMetaDataRecord(
             database_id=database_id, table_oid=table_oid, attnum=8,
             bool_input="checkbox", bool_true="true", bool_false="false",
-            num_min_frac_digits=2, num_max_frac_digits=8, num_show_as_perc=True,
+            num_min_frac_digits=2, num_max_frac_digits=8, num_grouping="force-no",
             mon_currency_symbol="$", mon_currency_location="after-minus",
             time_format=None, date_format=None,
-            duration_min=None, duration_max=None, duration_show_units=True,
+            duration_min=None, duration_max=None, num_format="german",
         ),
     ]
     actual_metadata_list = metadata.list_(table_oid=table_oid, database_id=database_id)

mathesar/tests/rpc/test_endpoints.py

@@ -164,6 +164,11 @@
         "explorations.delete",
         [user_is_authenticated]
     ),
+    (
+        explorations.run,
+        "explorations.run",
+        [user_is_authenticated]
+    ),
     (
         roles.list_,
         "roles.list",