[dataquery] New Data Query Module (#8907)

This implements a completely new, CouchDB free data query module based on the LORIS data query framework and the API introduced in PR#8268.

In the new module, all queries that are run by the user are saved into a history. The user can star or name a query in order to make it easier to find, rather than having to decide ahead of time that they want to save or share a query. Admins can pin queries (either to the top of the module or to the LORIS dashboard).

The UI is also (generally) simplified and more context-sensitive. The "define filters" tab should be easier to understand. The view data tab has a new view where candidates are rows, fields are columns, and session-scoped variables are displayed inline in the cell in a table.

Permissions are enforced for modules, candidates, and sessions.

Makefile

@@ -62,6 +62,9 @@ data_release:
 instrument_manager:
 	target=instrument_manager npm run compile
 
+dataquery:
+	target=dataquery npm run compile
+
 login:
 	target=login npm run compile
 

SQL/0000-00-02-Permission.sql

@@ -131,7 +131,8 @@ INSERT INTO `permissions` VALUES
     (60,'behavioural_quality_control_view','Flagged Behavioural Entries',(SELECT ID FROM modules WHERE Name='behavioural_qc'),'View','2'),
     (61,'api_docs','API documentation',(SELECT ID FROM modules WHERE Name='api_docs'),'View','2'),
     (62,'electrophysiology_browser_edit_annotations','Annotations',(SELECT ID FROM modules WHERE Name='electrophysiology_browser'),'Create/Edit','2'),
-    (63,'monitor_eeg_uploads','Monitor EEG uploads',(SELECT ID FROM modules WHERE Name='electrophysiology_uploader'),NULL,'2');
+    (63,'monitor_eeg_uploads','Monitor EEG uploads',(SELECT ID FROM modules WHERE Name='electrophysiology_uploader'),NULL,'2'),
+    (64,'dataquery_admin','Admin dataquery queries',(SELECT ID FROM modules WHERE Name='dataquery'),NULL,'2');
 
 INSERT INTO `user_perm_rel` (userID, permID)
   SELECT u.ID, p.permID

SQL/New_patches/2023-12-02-DQT-AdminPermission.sql

@@ -0,0 +1,6 @@
+INSERT INTO permissions (code, description, moduleID)
+    VALUES (
+      'dataquery_admin',
+      'Dataquery Admin',
+      (SELECT ID FROM modules WHERE Name='dataquery')
+    );

jsx/DataTable.d.ts

@@ -0,0 +1,70 @@
+import {ReactNode} from 'react';
+
+type TableRow = (string|null)[]
+
+type Field = {
+    show: boolean
+    label: string
+}
+
+type hideOptions = {
+    rowsPerPage: boolean
+    downloadCSV: boolean
+    defaultColumn: boolean
+}
+type DataTableProps = {
+    data: TableRow[]
+    rowNumLabel?: string
+    getFormattedCell: (label: string,
+        data: string,
+        row: TableRow,
+        headers: string[],
+        fieldNo: number) => ReactNode
+    onSort?: () => void
+    hide?: hideOptions
+    fields: Field[]
+    nullTableShow?: boolean
+    noDynamicTable?: boolean
+    getMappedCell?: (
+        label: string,
+        data: string|null,
+        row: TableRow,
+        headers: string[],
+        fieldNo: number) => string|(string|null)[]|null
+}
+
+/**
+ * The DataTable class. See DataTable.js
+ */
+class DataTable {
+    props: DataTableProps
+    state: any
+    context: object
+    refs: {[key: string]: ReactInstance}
+
+    /**
+     * Construct a new modal
+     */
+    constructor(props: DataTableProps)
+
+    /**
+     * React lifecycle method
+     *
+     * @returns {ReactNode}
+     */
+    render(): ReactNode
+
+    /**
+     * React lifecycle method
+     *
+     * @param {object} newstate - the state to overwrite
+     */
+    setState(newstate: object): void
+
+    /**
+     * React lifecycle method
+     */
+    forceUpdate(): void
+}
+
+export default DataTable;

jsx/DataTable.js

@@ -114,13 +114,22 @@ class DataTable extends Component {
    *
    * @param {number[]} filteredRowIndexes - The filtered Row Indexes
    */
+
   downloadCSV(filteredRowIndexes) {
     let csvData = filteredRowIndexes.map((id) => this.props.data[id]);
     // Map cell data to proper values if applicable.
     if (this.props.getMappedCell) {
       csvData = csvData
       .map((row, i) => this.props.fields
-        .map((field, j) => this.props.getMappedCell(field.label, row[j]))
+        .flatMap((field, j) => this.props.getMappedCell(
+            field.label,
+            row[j],
+            row,
+            this.props.fields.map(
+                (val) => val.label,
+            ),
+            j
+        ))
       );
     }
 
@@ -497,12 +506,18 @@ class DataTable extends Component {
             this.props.fields
               .forEach((field, k) => row[field.label] = rowData[k]);
 
+            const headers = this.props.fields.map(
+                (val) => val.label
+            );
+
             // Get custom cell formatting if available
             if (this.props.getFormattedCell) {
                 cell = this.props.getFormattedCell(
                     this.props.fields[j].label,
                     celldata,
-                    row
+                    row,
+                    headers,
+                    j
                 );
             } else {
                 cell = <td>{celldata}</td>;

jsx/Panel.d.ts

@@ -0,0 +1,53 @@
+import {ReactNode} from 'react';
+
+type PanelProps = {
+    initCollapsed?: boolean
+    collapsed?: boolean
+    parentId?: string
+    id?: string
+    height?: string
+    title?: string
+    class?: string
+    children: ReactNode
+    views?: object
+    collapsing?: boolean
+    bold?: boolean
+    panelSize?: string
+    style?: React.CSSProperties
+}
+
+/**
+ * The Modal class. See Modal.js
+ */
+class Panel {
+    props: PanelProps
+    state: any
+    context: object
+    refs: {[key: string]: ReactInstance}
+
+    /**
+     * Construct a new modal
+     */
+    constructor(props: PanelProps)
+
+    /**
+     * React lifecycle method
+     *
+     * @returns {ReactNode}
+     */
+    render(): ReactNode
+
+    /**
+     * React lifecycle method
+     *
+     * @param {object} newstate - the state to overwrite
+     */
+    setState(newstate: object): void
+
+    /**
+     * React lifecycle method
+     */
+    forceUpdate(): void
+}
+
+export default Panel;

modules/dataquery/jsx/calcpayload.tsx

@@ -0,0 +1,51 @@
+import {QueryTerm, QueryGroup} from './querydef';
+import {
+    APIQueryObject,
+    APIQueryField,
+    APIQueryGroupField,
+    APIQueryCriteriaGroup,
+} from './types';
+/**
+ * Calculates the payload to submit to the search endpoint
+ * to run the query.
+ *
+ * @param {APIQueryField[]} fields - the fields to query
+ * @param {QueryGroup} filters - the root of the filters
+ * @returns {APIQueryObject} - The query to send to the API
+ */
+export function calcPayload(
+    fields: APIQueryField[],
+    filters: QueryGroup
+): APIQueryObject {
+    const payload: APIQueryObject = {
+        type: 'candidates',
+        fields: fields.map((val: APIQueryField) => {
+            const fieldpayload: APIQueryField = {
+                module: val.module,
+                category: val.category,
+                field: val.field,
+            };
+            if (val.visits) {
+                fieldpayload.visits = val.visits;
+            }
+            return fieldpayload;
+        },
+        ),
+    };
+    if (filters.group.length > 0) {
+        payload.criteria = {
+            operator: filters.operator,
+            group: filters.group.map( (val) => {
+                if (val instanceof QueryTerm) {
+                    return val as APIQueryGroupField;
+                } else if (val instanceof QueryGroup) {
+                    return val as APIQueryCriteriaGroup;
+                } else {
+                    throw new Error('Invalid query');
+                }
+            }),
+        };
+    }
+    return payload;
+}
+

modules/dataquery/jsx/components/expansionpanels.tsx

@@ -0,0 +1,37 @@
+import React from 'react';
+import Panel from 'jsx/Panel';
+
+/**
+ * Render a series of expansion panels
+ *
+ * @param {object} props - React props
+ * @param {boolean?} props.alwaysOpen - If true, panels can not be toggled
+ * @param {object} props.panels - Array of individual panels
+ * @returns {React.ReactElement} - The panels
+ */
+const ExpansionPanels = (props: {
+    alwaysOpen?: boolean,
+    panels: {
+        title: string,
+        content: React.ReactElement,
+        defaultOpen?: boolean,
+        alwaysOpen: boolean,
+    }[]
+}) => {
+  return (
+    <div className={'container-fluid'}
+         style={{margin: '0 auto', maxWidth: '900px'}}>
+      { props.panels.map((panel, index) => (
+        <Panel
+          key={index}
+          title={panel.title}
+          collapsed={panel.alwaysOpen}
+          initCollapsed={panel.defaultOpen || props.alwaysOpen || true}>
+          {panel.content}
+        </Panel>
+      ))}
+    </div>
+  );
+};
+
+export default ExpansionPanels;

modules/dataquery/jsx/components/filterableselectgroup.tsx

@@ -0,0 +1,90 @@
+import Select, {SingleValue} from 'react-select';
+
+type SelectOption = {
+    label: string,
+    value: string,
+    module: string,
+};
+
+type SelectGroup = {
+    label: string,
+    options: SelectOption[],
+};
+/**
+ * Render a select with option groups that can be
+ * filtered
+ *
+ * @param {object} props - react props
+ * @param {function} props.onChange - Callback on selection change
+ * @param {string?} props.placeholder - An optional placeholder value when no elements are selected
+ * @param {object} props.groups - Groups to select the dropdown into
+ * @param {function} props.mapGroupName - A mapper from backend to frontend name for groups
+ * @returns {React.ReactElement} - The element
+ */
+function FilterableSelectGroup(props: {
+    onChange: (module: string, value: string) => void,
+    placeholder?: string,
+    groups: object,
+    mapGroupName?: (module: string) => string,
+}) {
+    const groups: SelectGroup[] = [];
+    const placeholder = props.placeholder || 'Select a category';
+    for (const [module, subcategories]
+        of Object.entries(props.groups)) {
+      const options: SelectOption[] = [];
+      for (const [value, desc]
+          of Object.entries(subcategories) as unknown as [string, string]) {
+        options.push({
+          value: value,
+          label: desc,
+          module: module,
+        });
+      }
+
+      let label = module;
+      if (props.mapGroupName) {
+        label = props.mapGroupName(module);
+      }
+      groups.push({
+        label: label,
+        options: options,
+      });
+    }
+
+    /**
+     * Callback to call when the selection changes.
+     *
+     * @param {object} e - The click event callback
+     * @param {string} e.module - The module
+     * @param {string} e.value - The value
+     * @returns {void}
+     */
+    const selected = (e: SingleValue<SelectGroup>) => {
+        // The callback should be (e: SelectOption) but typescript
+        // is convinced that it's a SingleValue<SelectGroup>.
+        // console.log(e) confirms that it has the same structure
+        // as SelectOption, so just convert it and explicitly
+        // cast it unsafely to make the compiler happy.
+        const val: SelectOption = e as unknown as SelectOption;
+        props.onChange(val.module, val.value);
+    };
+    return (
+        <div>
+            <Select options={groups} onChange={selected}
+                 menuPortalTarget={document.body}
+                 styles={{menuPortal:
+                    /**
+                     * Add a z-index to ensure the element stays visible
+                     *
+                     * @param {object} base - the base CSS
+                     * @returns {object} - the new CSS with z-index added
+                     */
+                    (base) => ({...base, zIndex: 9999})}
+                }
+                 placeholder={placeholder}
+            />
+        </div>
+    );
+}
+
+export default FilterableSelectGroup;