Skip to content

Commit

Permalink
Merge branch 'develop' into records_add
Browse files Browse the repository at this point in the history
  • Loading branch information
mathemancer authored Aug 9, 2024
2 parents ca70534 + c3bbfce commit a1044f0
Show file tree
Hide file tree
Showing 11 changed files with 233 additions and 64 deletions.
4 changes: 2 additions & 2 deletions db/sql/10_msar_joinable_tables.sql
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ jsonb AS $$
SELECT jsonb_object_agg(tt_oid, tt_info) AS tt FROM target_cte
)
SELECT jsonb_build_object(
'joinable_tables', joinable_tables.jt,
'target_table_info', target_table_info.tt
'joinable_tables', COALESCE(joinable_tables.jt, '[]'::jsonb),
'target_table_info', COALESCE(target_table_info.tt, '{}'::jsonb)
) FROM joinable_tables, target_table_info;
$$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT;
3 changes: 2 additions & 1 deletion mathesar_ui/src/api/rest/types/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { PaginatedResponse } from '@mathesar/api/rest/utils/requestUtils';
import type { Column } from '@mathesar/api/rpc/columns';
import type { Schema } from '@mathesar/api/rpc/schemas';
import type { JoinPath } from '@mathesar/api/rpc/tables';
import type { FilterId } from '@mathesar/stores/abstract-types/types';

export type QueryColumnAlias = string;

Expand All @@ -21,7 +22,7 @@ type FilterConditionParams = [
{ column_name: [string] },
...{ literal: [unknown] }[],
];
type FilterCondition = Record<string, FilterConditionParams>;
type FilterCondition = Partial<Record<FilterId, FilterConditionParams>>;

export interface QueryInstanceFilterTransformation {
type: 'filter';
Expand Down
51 changes: 40 additions & 11 deletions mathesar_ui/src/api/rpc/records.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,45 @@ export interface SortingEntry {
attnum: number;
direction: SortDirection;
}
export type FilterCombination = 'and' | 'or';
export type FilterConditionParams = [
{ column_id: [number] },
...{ literal: [unknown] }[],
];
export type FilterCondition = Record<string, FilterConditionParams>;
type MakeFilteringOption<U> = U extends string
? { [k in U]: FilterRequest[] }
: never;
type FilterRequest = FilterCondition | MakeFilteringOption<FilterCombination>;

export interface SqlComparison {
type:
| 'and'
| 'or'
| 'equal'
| 'lesser'
| 'greater'
| 'lesser_or_equal'
| 'greater_or_equal'
| 'contains_case_insensitive'
| 'contains'
| 'starts_with'
| 'json_array_contains';
args: [SqlExpr, SqlExpr];
}

export interface SqlFunction {
type:
| 'null'
| 'not_null'
| 'json_array_length'
| 'uri_scheme'
| 'uri_authority'
| 'email_domain';
args: [SqlExpr];
}

export interface SqlLiteral {
type: 'literal';
value: string | number | null;
}

export interface SqlColumn {
type: 'attnum';
value: number;
}

export type SqlExpr = SqlComparison | SqlFunction | SqlLiteral | SqlColumn;

export interface RecordsListParams {
database_id: number;
Expand All @@ -45,7 +74,7 @@ export interface RecordsListParams {
offset?: number;
order?: SortingEntry[];
group?: Pick<Grouping, 'columns' | 'preproc'>;
filter?: FilterRequest;
filter?: SqlExpr;
search_fuzzy?: Record<string, unknown>[];
}

Expand Down
11 changes: 7 additions & 4 deletions mathesar_ui/src/components/filter-entry/FilterEntry.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
import { getDbTypeBasedInputCap } from '@mathesar/components/cell-fabric/utils';
import ColumnName from '@mathesar/components/column/ColumnName.svelte';
import { iconDeleteMajor } from '@mathesar/icons';
import type { AbstractTypeFilterDefinition } from '@mathesar/stores/abstract-types/types';
import type {
AbstractTypeFilterDefinition,
FilterId,
} from '@mathesar/stores/abstract-types/types';
import type RecordSummaryStore from '@mathesar/stores/table-data/record-summaries/RecordSummaryStore';
import type { RecordSummariesForColumn } from '@mathesar/stores/table-data/record-summaries/recordSummaryUtils';
import type { ReadableMapLike } from '@mathesar/typeUtils';
Expand Down Expand Up @@ -37,7 +40,7 @@
) => ConstraintType[] | undefined = () => undefined;
export let columnIdentifier: ColumnLikeType['id'] | undefined;
export let conditionIdentifier: string | undefined;
export let conditionIdentifier: FilterId | undefined;
export let value: unknown | undefined;
export let layout: 'horizontal' | 'vertical' = 'horizontal';
Expand Down Expand Up @@ -106,7 +109,7 @@
return undefined;
}
function getConditionName(_conditionId?: string) {
function getConditionName(_conditionId?: FilterId) {
if (_conditionId) {
return selectedColumnFiltersMap.get(_conditionId)?.name ?? '';
}
Expand Down Expand Up @@ -166,7 +169,7 @@
dispatch('update');
}
function onConditionChange(_conditionId?: string) {
function onConditionChange(_conditionId?: FilterId) {
if (!_conditionId) {
return;
}
Expand Down
44 changes: 43 additions & 1 deletion mathesar_ui/src/stores/abstract-types/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,50 @@ export interface AbstractTypesSubstance {
error?: string;
}

/**
* These filter ids represent the filter functions used for the _old_ filtering
* system (circa 2023). The UI is still designed around these filter functions
* which is why we still have code for it within the front end.
*
* In 2024 we moved filtering logic from the service layer into the DB layer and
* introduced a new filtering system that is more flexible. The new filtering
* system is much more flexible and can handle complex filtering expressions
* with arbitrary nesting.
*
* Elsewhere in the front end codebase, we have a compatibility layer that
* translates between the old filtering system and the new filtering system.
* Search for `filterEntryToSqlExpr` to find that compatibility layer.
*
* If at some point we decide to design a more flexible user-facing filtering
* UI, then we could model that UI (and the resulting front end data structures)
* around the `SqlExpr` data structure. This would allow us to avoid the need to
* maintain the type below because we would be able to directly support the
* filtering expressions that the API expects.
*/
export type FilterId =
| 'contains_case_insensitive'
| 'email_domain_contains'
| 'email_domain_equals'
| 'equal'
| 'greater_or_equal'
| 'greater'
| 'json_array_contains'
| 'json_array_length_equals'
| 'json_array_length_greater_or_equal'
| 'json_array_length_greater_than'
| 'json_array_length_less_or_equal'
| 'json_array_length_less_than'
| 'json_array_not_empty'
| 'lesser_or_equal'
| 'lesser'
| 'not_null'
| 'null'
| 'starts_with_case_insensitive'
| 'uri_authority_contains'
| 'uri_scheme_equals';

export interface AbstractTypeFilterDefinitionResponse {
id: string;
id: FilterId;
name: string;
aliases?: Record<AbstractTypeCategoryIdentifier, string>;
uiTypeParameterMap: Partial<
Expand Down
18 changes: 9 additions & 9 deletions mathesar_ui/src/stores/databases.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,13 @@ class DatabasesStore {
export const databasesStore: MakeWritablePropertiesReadable<DatabasesStore> =
new DatabasesStore();

/**
* @throws an error when used in a context where no current database exists.
* This behavior sacrifices some stability for the sake of developer ergonomics.
* This sacrifice seems acceptable given that such a large part of the
* application depends on the existence of one and only one database.
/** ⚠️ This readable store contains a type assertion designed to sacrifice type
* safety for the benefit of convenience.
*
* We need to access `currentDatabase` like EVERYWHERE throughout the app, and
* we'd like to avoid checking if it's defined every time. So we assert that it
* is defined, and we'll just have to be careful to **never use the value from
* this store within a context where no database is set.**
*/
export const currentDatabase = derived(databasesStore.currentDatabase, (c) => {
if (!c) throw new Error('No current database');
return c;
});
export const currentDatabase =
databasesStore.currentDatabase as Readable<Database>;
125 changes: 108 additions & 17 deletions mathesar_ui/src/stores/table-data/filtering.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,103 @@
import type {
FilterCombination,
FilterCondition,
FilterConditionParams,
RecordsListParams,
SqlColumn,
SqlComparison,
SqlExpr,
SqlFunction,
SqlLiteral,
} from '@mathesar/api/rpc/records';

import type { FilterId } from '../abstract-types/types';

export type FilterCombination = 'and' | 'or';

export interface FilterEntry {
readonly columnId: number;
readonly conditionId: string;
readonly conditionId: FilterId;
readonly value: unknown;
}

function makeApiFilterCondition(filterEntry: FilterEntry): FilterCondition {
const params: FilterConditionParams = [{ column_id: [filterEntry.columnId] }];
if (typeof filterEntry.value !== 'undefined') {
params.push({ literal: [filterEntry.value] });
/**
* This function is glue code between the old filtering system (circa 2023) and
* the new filtering system (circa 2024).
*
* At the API layer, the old filtering system was relatively flat and not very
* flexible.
*
* The new filtering system is much more flexible and can handle complex
* filtering expressions with arbitrary nesting. Brent introduced this new
* system when moving the filtering logic from the service layer to the DB
* layer. As a result, the API changed. But in an effort to minimize disturbance
* within the front end, we introduced this compatibility layer.
*
* The `FilterEntry` argument to this function is a data structure from the
* _old_ filtering system. It is consistent with the behavior of the UI and the
* data structures used for filtering throughout the front end. The `SqlExpr`
* return value is a data structure from the _new_ filtering system. It is
* consistent with the behavior of the API.
*
* If, at some point, we'd like to design a more flexible user-facing filtering
* UI, then we could model that UI (and the resulting front end data structures)
* around the `SqlExpr` data structure. This would allow us to avoid the need
* for this compatibility layer.
*/
function filterEntryToSqlExpr(filterEntry: FilterEntry): SqlExpr {
const column: SqlColumn = { type: 'attnum', value: filterEntry.columnId };

/** Generate an SqlLiteral value */
function value(v = filterEntry.value): SqlLiteral {
return { type: 'literal', value: String(v) };
}

/** Generate an SqlComparison */
function cmp(
type: SqlComparison['type'],
args: [SqlExpr, SqlExpr] = [column, value()],
): SqlComparison {
return { type, args };
}

/** Generate an SqlFunction */
function fn(type: SqlFunction['type'], arg: SqlExpr = column): SqlFunction {
return { type, args: [arg] };
}

/** Generate an SqlComparison of a JSON array length vs a value */
function arrayLengthCmp(
type: SqlComparison['type'],
v = value(),
): SqlComparison {
return cmp(type, [fn('json_array_length'), v]);
}
return { [filterEntry.conditionId]: params };

const compatibilityMap: Record<FilterId, SqlExpr> = {
contains_case_insensitive: cmp('contains_case_insensitive'),
email_domain_contains: cmp('contains', [fn('email_domain'), value()]),
email_domain_equals: cmp('equal', [fn('email_domain'), value()]),
equal: cmp('equal'),
greater_or_equal: cmp('greater_or_equal'),
greater: cmp('greater'),
json_array_contains: cmp('json_array_contains'),
json_array_length_equals: arrayLengthCmp('equal'),
json_array_length_greater_or_equal: arrayLengthCmp('greater_or_equal'),
json_array_length_greater_than: arrayLengthCmp('greater'),
json_array_length_less_or_equal: arrayLengthCmp('lesser_or_equal'),
json_array_length_less_than: arrayLengthCmp('lesser'),
json_array_not_empty: arrayLengthCmp('greater', value(0)),
lesser_or_equal: cmp('lesser_or_equal'),
lesser: cmp('lesser'),
not_null: fn('not_null'),
null: fn('null'),
starts_with_case_insensitive: cmp('starts_with'),
uri_authority_contains: cmp('contains', [fn('uri_authority'), value()]),
uri_scheme_equals: cmp('equal', [fn('uri_scheme'), value()]),
};

return compatibilityMap[filterEntry.conditionId];
}

/** [ columnId, operation, value ] */
type TerseFilterEntry = [number, string, unknown];
type TerseFilterEntry = [number, FilterId, unknown];

function makeTerseFilterEntry(filterEntry: FilterEntry): TerseFilterEntry {
return [filterEntry.columnId, filterEntry.conditionId, filterEntry.value];
Expand All @@ -39,6 +116,14 @@ export const defaultFilterCombination = filterCombinations[0];

export type TerseFiltering = [FilterCombination, TerseFilterEntry[]];

/**
* The data structure here is designed to model the behavior of the UI, however
* the API has a different (and much more flexible structure). When we first
* wrote this code, the UI and the API had similar structures, so the structure
* used in this class fit well with both. However, now that the API has changed,
* it might be worth using the (more flexible) data structure within this class
* should the need ever arise for any substantial refactoring here.
*/
export class Filtering {
combination: FilterCombination;

Expand Down Expand Up @@ -118,16 +203,22 @@ export class Filtering {
}

recordsRequestParams(): Pick<RecordsListParams, 'filter'> {
if (!this.entries.length) {
if (this.entries.length === 0) {
return {};
}
const conditions = this.entries.map(makeApiFilterCondition);
if (conditions.length === 1) {
return { filter: conditions[0] };
if (this.entries.length === 1) {
return { filter: filterEntryToSqlExpr(this.entries[0]) };
}
const filter =
this.combination === 'and' ? { and: conditions } : { or: conditions };
return { filter };

let expr = filterEntryToSqlExpr(this.entries[0]);
for (let i = 1; i < this.entries.length; i += 1) {
expr = {
type: this.combination,
args: [expr, filterEntryToSqlExpr(this.entries[i])],
};
}

return { filter: expr };
}

terse(): TerseFiltering {
Expand Down
8 changes: 4 additions & 4 deletions mathesar_ui/src/stores/table-data/records.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ import {
} from '@mathesar-component-library';

import type { ColumnsDataStore } from './columns';
import type { Filtering } from './filtering';
import type { FilterEntry, Filtering } from './filtering';
import type { Grouping as GroupingRequest } from './grouping';
import type { Meta } from './meta';
import RecordSummaryStore from './record-summaries/RecordSummaryStore';
Expand Down Expand Up @@ -385,9 +385,9 @@ export class RecordsData {

try {
const params = get(this.meta.recordsRequestParamsData);
const contextualFilterEntries = [...this.contextualFilters].map(
([columnId, value]) => ({ columnId, conditionId: 'equal', value }),
);
const contextualFilterEntries: FilterEntry[] = [
...this.contextualFilters,
].map(([columnId, value]) => ({ columnId, conditionId: 'equal', value }));

const recordsListParams: RecordsListParams = {
...this.apiContext,
Expand Down
Loading

0 comments on commit a1044f0

Please sign in to comment.